Storage: add SS auth, log config. ref #9910 PR #22

 - Add SS API key authentication
 - Add log level configuration
 - Update README and sample script
 - Bug fix

Author: hakamine

README.md

@@ -20,6 +20,7 @@ The Automation Tools project is a set of python scripts, that are designed to au
     - [user-input](#user-input)
   - [Logs](#logs)
   - [Multiple automated transfer instances](#multiple-automated-transfer-instances)
+- [Automated package moving](#automated-package-moving)
 - [Related Projects](#related-projects)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -199,6 +200,37 @@ You may need to set up multiple automated transfer instances, for example if req
 
 In case different hooks are required for each instance, a possible approach is to checkout a new instance of the automation tools, for example in `/usr/lib/archivematica/automation-tools-2`
 
+
+Automated package moving
+------------------------
+
+`storage/move_packages.py` is a helper script used to automate moving packages between locations.
+
+The script takes the UUID of a `<move_from>` Location, and the UUID of a `<move_to>` Location.
+
+When executed, the script will:
+
+* query the storage service and ask for a list of packages in the move_from Location. 
+* check if the first package returned is in the automation tools db. 
+* If it is not there, the script will call the move_package endpoint with this packages’s UUID and the UUID of the move_to Location, then exit. 
+* The next time the script is executed, it will query the status of the package. 
+* If it is ‘moving’, the script will go back to sleep. 
+* Once the status of the current package is no longer ‘moving’, the script will go on to the next package.
+
+The script makes use of the `move` endpoint in the Storage Service REST API. 
+The `move` endpoint takes two arguments: UUID of an existing package (AIP or DIP or transfer) and the UUID of a Location. 
+
+The move_package endpoint will:
+* Confirm that the type of package (AIP or DIP or Transfer) matches the new Location
+* Set the status of the package to ‘moving’
+* Copy the package from its current location to the new location using rsync and leave the original copy of the package alone
+* Execute any post store hooks configured for the Location (for example, call the Arkivum finalize command)
+* Update the internal storage service database with the new location of the package (and new status, set by the Space)
+* If the rsync command does not work or there is a failure in post store commands, the status of the package will be set to ‘move failed’, and the internal ss database will not be updated
+
+The `etc` directory contains an example script (`storage-script.sh`) and config file (`storage.conf`)
+
+
 Related Projects
 ----------------
 

common/utils.py

@@ -22,7 +22,7 @@ def get_setting(config_file, config_name, setting, default=None):
         return default
 
 
-def configure_logging(name, filename):
+def configure_logging(name, filename, loglevel):
     """
     Configure logging
 
@@ -54,7 +54,7 @@ def configure_logging(name, filename):
         },
         'loggers': {
             name: {
-                'level': 'INFO',  # One of INFO, DEBUG, WARNING, ERROR, CRITICAL
+                'level': loglevel,  # One of INFO, DEBUG, WARNING, ERROR, CRITICAL
                 'handlers': ['console', 'file'],
             },
         },
@@ -73,10 +73,10 @@ def open_pid_file(pid_file, logger=None):
     try:
         # Open PID file only if it doesn't exist for read/write
         f = os.fdopen(os.open(pid_file, os.O_CREAT | os.O_EXCL | os.O_RDWR), 'r+')
-    except OSError as e:
-         if logger:
-             logger.info('Error accessing pid file %s: %s', pid_file, exc_info=True)
-         return None
+    except OSError:
+        if logger:
+            logger.info('Error accessing pid file %s:', pid_file, exc_info=True)
+        return None
     except Exception:
         if logger:
             logger.info('This script is already running. To override this behaviour and start a new run, remove %s', pid_file)

etc/storage-script.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# storage move package script example
+# /etc/archivematica/automation-tools/storage-script.sh
+cd /usr/lib/archivematica/automation-tools/
+/usr/share/python/automation-tools/bin/python -m storage.move_packages --config-file /etc/archivematica/automation-tools/storage.conf --ss-user USERNAME --ss-api-key KEY --from-location a13e466d-a144-430a-85b3-95e6aaa52f20  --to-location fbdf5325-c342-406a-ba66-3f4e3f73cf5f

storage/move_packages.py

@@ -12,7 +12,6 @@
 import logging.config  # Has to be imported separately
 import os
 import requests
-from six.moves import configparser
 import sys
 
 # This project
@@ -30,21 +29,23 @@ def get_setting(setting, default=None):
     return utils.get_setting(CONFIG_FILE, 'storage', setting, default)
 
 
-def setup(config_file):
+def setup(config_file, log_level):
     global CONFIG_FILE
     CONFIG_FILE = config_file
 
     # Configure logging
     default_logfile = os.path.join(THIS_DIR, 'automate-storage.log')
     logfile = get_setting('logfile', default_logfile)
-    utils.configure_logging('storage', logfile)
+    utils.configure_logging('storage', logfile, log_level)
 
 
-def get_first_eligible_package_in_location(ss_url, location_uuid):
+def get_first_eligible_package_in_location(ss_url, ss_user, ss_api_key, location_uuid):
     """
     Get first package in a location that has a status of either UPLOADED or MOVING.
 
     :param str ss_url: Storage service URL
+    :param ss_user: User on the Storage Service for authentication
+    :param ss_api_key: API key for user on the Storage Service for authentication
     :param str location_uuid: UUID of location to fetch package details from
     :returns: Dict containing package details or None if none found
     """
@@ -55,7 +56,9 @@ def get_first_eligible_package_in_location(ss_url, location_uuid):
         ("current_location__uuid", location_uuid),
         ("status__in", "UPLOADED"),
         ("status__in", "MOVING"),
-        ("order_by", "uuid")]
+        ("order_by", "uuid"),
+        ("username", ss_user),
+        ("api_key", ss_api_key)]
 
     result = utils.call_url_json(get_url, params, LOGGER)
     if 'objects' in result and len(result['objects']):
@@ -64,22 +67,31 @@ def get_first_eligible_package_in_location(ss_url, location_uuid):
         return None
 
 
-def move_to_location(ss_url, package_uuid, location_uuid):
+def move_to_location(ss_url, ss_user, ss_api_key, package_uuid, location_uuid):
     """
     Send request to move package to another location.
 
     :param str ss_url: Storage service URL
+    :param ss_user: User on the Storage Service for authentication
+    :param ss_api_key: API key for user on the Storage Service for authentication
     :param str package_uuid: UUID of package to move
     :param str location_uuid: UUID of location to move package to
     :returns: Dict representing JSON response.
     """
     LOGGER.info("Moving package %s to location %s", package_uuid, location_uuid)
 
     post_url = '%s/api/v2/file/%s/move/' % (ss_url, package_uuid)
-    post_data = {'location_uuid': location_uuid}
+    params = {
+        'username': ss_user,
+        'api_key': ss_api_key,
+    }
+    post_data = {
+        'location_uuid': location_uuid,
+    }
     LOGGER.debug('URL: %s; Body: %s;', post_url, json.dumps(post_data))
 
     r = requests.post(post_url,
+                      params=params,
                       json=post_data,
                       headers={'content-type': 'application/json'})
     LOGGER.debug('Response: %s', r)
@@ -90,9 +102,9 @@ def move_to_location(ss_url, package_uuid, location_uuid):
     return r.json()
 
 
-def main(ss_url, from_location_uuid, to_location_uuid, config_file=None):
+def main(ss_url, ss_user, ss_api_key, from_location_uuid, to_location_uuid, config_file=None, log_level='INFO'):
 
-    setup(config_file)
+    setup(config_file, log_level)
 
     LOGGER.info("Waking up")
 
@@ -104,14 +116,14 @@ def main(ss_url, from_location_uuid, to_location_uuid, config_file=None):
 
     # Check statuis of last package and attempt move
     move_result = None
-    package = get_first_eligible_package_in_location(ss_url, from_location_uuid)
+    package = get_first_eligible_package_in_location(ss_url, ss_user, ss_api_key, from_location_uuid)
     if package is None:
         LOGGER.info('No packages remain in location, nothing to do.')
     elif package['status'] == 'MOVING':
         LOGGER.info('Current package %s still processing, nothing to do.', package['uuid'])
     else:
         LOGGER.info('Moving package %s.', package['uuid'])
-        move_result = move_to_location(ss_url, package['uuid'], to_location_uuid)
+        move_result = move_to_location(ss_url, ss_user, ss_api_key, package['uuid'], to_location_uuid)
         if move_result is None:
             LOGGER.info('Move request failed')
         else:
@@ -125,14 +137,20 @@ def main(ss_url, from_location_uuid, to_location_uuid, config_file=None):
 
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     parser.add_argument('--ss-url', '-s', metavar='URL', help='Storage Service URL. Default: http://127.0.0.1:8000', default='http://127.0.0.1:8000')
+    parser.add_argument('--ss-user', metavar='USERNAME', required=True, help='Username of the Storage Service user to authenticate as.')
+    parser.add_argument('--ss-api-key', metavar='KEY', required=True, help='API key of the Storage Service user.')
     parser.add_argument('--from-location', '-f', metavar='SOURCE', help="UUID of source location.", required=True)
     parser.add_argument('--to-location', '-t', metavar='DEST', help="UUID of destination location.", required=True)
     parser.add_argument('--config-file', '-c', metavar='FILE', help='Configuration file(log/db/PID files)', default=None)
+    parser.add_argument('--log-level', choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], default='INFO', help='Set the debugging output level.')
     args = parser.parse_args()
 
     sys.exit(main(
         ss_url=args.ss_url,
+        ss_user=args.ss_user,
+        ss_api_key=args.ss_api_key,
         from_location_uuid=args.from_location,
         to_location_uuid=args.to_location,
-        config_file=args.config_file
+        config_file=args.config_file,
+        log_level=args.log_level,
     ))