This topic provides various methods for batch deletion of objects from a bucket.

I. Using Client Tools

Using ZOSBrowser to batch delete objects from a bucket

Download the ZOSBrowser tool from the official eSurfing Cloud website. For more information about how to use the tool, see ZOSbrowser User Guide.

Deleting multiple objects

When using ZOSBrowser, upon selecting a bucket, some or all objects within the bucket are displayed in a list. Select the checkboxes next to the objects that you want to delete, then click Delete below the list. In the dialog box that appears, click OK to batch delete these objects.

If a selected object is a file folder, all files in the file folder will be deleted.

Deleting all objects

Click the checkbox in the first column of the header to select all objects. A maximum of 1000 objects can be selected at once.

Click Delete . In the dialog box that appears, click OK to delete them all.

2. Using S3Browser to batch delete objects from a bucket

Download the S3Browser client tool from the official S3Browser website. You can use the non-commercial version upon installation.

For more information about how to use this tool, see the related documents available on the official S3Browser website.

Deleting multiple objects

Press the Ctrl key while clicking objects in the object list to select them, then click Delete below the list. In the dialog box that appears, click Confirm to batch delete these objects.

Deleting all objects

Choose Files from the S3Browser menu, click Select All to select all the objects displayed. A maximum of 1000 objects can be displayed in the list.

Click Delete below the list. In the dialog box that appears, click Confirm to delete them all.

II. Using the SDK for API Invocation

See Developer Documentation to select the appropriate SDK. Configure the environmentand invoke an API to delete objects.

Using the Python

SDK Python SDK is used as an example to describe how to invoke an API to delete all objects from a bucket. For more information about the Python SDK, see ZOS Python_SDK User Guide.

config.py

Bucket_name="testbk"
AK="test"
SK="test"
URL="http://192.168.16.155:8000"
multi_pool_num=16
loglevel="debug"

log.py

#-*-coding:utf-8 -*-
from boto3.session import Session
from botocore.client import Config
import config
import log
import os
import time
import logging
import threading
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED 
 
 
from multiprocessing import Pool
from time import strftime, localtime
 
def err_call(value):
    logger = logging.getLogger(str(os.getpid()))
    logger.error("____err_call____ {0}".format(value))
 
 
def delete_sub_objs(res_contents, s3_client, logger):
  for obj in res_contents:
    key_name = obj.get("Key")
    try:
      s3_client.delete_object(
        Bucket=config.Bucket_name,
        Key=key_name
      )
      logger.debug("delete obj {0} complete...".format(key_name))
    except Exception as e:
      logger.error('__file__:{0}, objname:{1}, lineno:{2}'.format(e.__traceback__.tb_frame.f_globals['__file__'], key_name, e.__traceback__.tb_lineno))
      logger.error(e.args)
 
def delete_objs(res_contents):
  session = Session(config.AK, config.SK)
  s3_client = session.client('s3', endpoint_url=config.URL, verify=False)
  log.create_log_path(config.loglevel)
  a_logger = logging.getLogger(str(os.getpid()))
  total_nums = len(res_contents)
  begin = 0
  remainder_nums = 0
  a_logger.info("begin delete_objs...")
  subentry_obj_nums = total_nums // 50
  if subentry_obj_nums < 5:
      subentry_obj_nums = 5
  else:
      remainder_nums = total_nums % 50
  future_list = []
  with ThreadPoolExecutor(max_workers=50) as pool:
      while begin < total_nums:
          if remainder_nums == 0:
              end = begin + subentry_obj_nums
          else:
              end = begin + subentry_obj_nums + 1
              remainder_nums = remainder_nums - 1
          entry = res_contents[begin:end]
          future = pool.submit(delete_sub_objs, entry, s3_client, a_logger)
          future_list.append(future)
          begin = end
      wait(future_list, return_when= ALL_COMPLETED)
  a_logger.info("end delete_objs..., total num: {}".format(total_nums))
 
if __name__ == "__main__":
  log.create_log_path(config.loglevel)
  s_logger = logging.getLogger(str(os.getpid()))
  pool = Pool(processes=config.multi_pool_num)
  session = Session(config.AK, config.SK)
  s3_client = session.client('s3', endpoint_url=config.URL, verify=False)
  key_marker = ""
  next_key_marker = ""
  is_truncated = True
  s_logger.info('=============================del begin time=============================== :{0}'.format(strftime("%Y-%m-%d %H:%M:%S", localtime())))
  while is_truncated:
    key_marker = next_key_marker
    res_obj = s3_client.list_objects(Bucket=config.Bucket_name, Marker=key_marker, MaxKeys=1000)
    res_contents = res_obj.get('Contents', [])
 
    s_logger.debug(res_contents)
    if res_contents:
      pool.apply_async(delete_objs, args=(res_contents,), error_callback = err_call)
 
      while len(pool._cache) >= config.multi_pool_num*2:
          s_logger.info('waiting for cache to clear...')
          time.sleep(3)
    is_truncated = res_obj.get('IsTruncated')
    next_key_marker = res_obj.get('NextMarker')
 
  pool.close()
  pool.join()
  s_logger.info('=============================del end time ============================= :{0}'.format(strftime("%Y-%m-%d %H:%M:%S", localtime())))

bucket_empty.py

#-*-coding:utf-8 -*-
from boto3.session import Session
from botocore.client import Config
import config
import log
import os
import time
import logging
import threading
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED 
 
 
from multiprocessing import Pool
from time import strftime, localtime
 
def err_call(value):
    logger = logging.getLogger(str(os.getpid()))
    logger.error("____err_call____ {0}".format(value))
 
 
def delete_sub_objs(res_contents, s3_client, logger):
  for obj in res_contents:
    key_name = obj.get("Key")
    try:
      s3_client.delete_object(
        Bucket=config.Bucket_name,
        Key=key_name
      )
      logger.debug("delete obj {0} complete...".format(key_name))
    except Exception as e:
      logger.error('__file__:{0}, objname:{1}, lineno:{2}'.format(e.__traceback__.tb_frame.f_globals['__file__'], key_name, e.__traceback__.tb_lineno))
      logger.error(e.args)
 
def delete_objs(res_contents):
  session = Session(config.AK, config.SK)
  s3_client = session.client('s3', endpoint_url=config.URL, verify=False)
  log.create_log_path(config.loglevel)
  a_logger = logging.getLogger(str(os.getpid()))
  total_nums = len(res_contents)
  begin = 0
  remainder_nums = 0
  a_logger.info("begin delete_objs...")
  subentry_obj_nums = total_nums // 50
  if subentry_obj_nums < 5:
      subentry_obj_nums = 5
  else:
      remainder_nums = total_nums % 50
  future_list = []
  with ThreadPoolExecutor(max_workers=50) as pool:
      while begin < total_nums:
          if remainder_nums == 0:
              end = begin + subentry_obj_nums
          else:
              end = begin + subentry_obj_nums + 1
              remainder_nums = remainder_nums - 1
          entry = res_contents[begin:end]
          future = pool.submit(delete_sub_objs, entry, s3_client, a_logger)
          future_list.append(future)
          begin = end
      wait(future_list, return_when= ALL_COMPLETED)
  a_logger.info("end delete_objs..., total num: {}".format(total_nums))
 
if __name__ == "__main__":
  log.create_log_path(config.loglevel)
  s_logger = logging.getLogger(str(os.getpid()))
  pool = Pool(processes=config.multi_pool_num)
  session = Session(config.AK, config.SK)
  s3_client = session.client('s3', endpoint_url=config.URL, verify=False)
  key_marker = ""
  next_key_marker = ""
  is_truncated = True
  s_logger.info('=============================del begin time=============================== :{0}'.format(strftime("%Y-%m-%d %H:%M:%S", localtime())))
  while is_truncated:
    key_marker = next_key_marker
    res_obj = s3_client.list_objects(Bucket=config.Bucket_name, Marker=key_marker, MaxKeys=1000)
    res_contents = res_obj.get('Contents', [])
 
    s_logger.debug(res_contents)
    if res_contents:
      pool.apply_async(delete_objs, args=(res_contents,), error_callback = err_call)
 
      while len(pool._cache) >= config.multi_pool_num*2:
          s_logger.info('waiting for cache to clear...')
          time.sleep(3)
    is_truncated = res_obj.get('IsTruncated')
    next_key_marker = res_obj.get('NextMarker')
 
  pool.close()
  pool.join()
  s_logger.info('=============================del end time ============================= :{0}'.format(strftime("%Y-%m-%d %H:%M:%S", localtime())))

Run python bucket_empty.py.

I. Using Client Tools

Using ZOSBrowser to batch delete objects from a bucket

Deleting multiple objects

Deleting all objects

2. Using S3Browser to batch delete objects from a bucket

Deleting multiple objects

Deleting all objects

II. Using the SDK for API Invocation

Using the Python

Zettabyte Object Storage

Batch Delete Bucket Objects