hashsync.py

#!/usr/local/bin/python3
#hashsync.py
#better file movement
#takes arguments for source or dest, source can be file or dir, dest must be dir
#copies files, hashes pre and post copy, deletes if hashes match, deletes dir if empty
#you can also:
#verify against another directory or the given directory (-v)
#copy files, don't delete from source directory (-c)
#quiet mode (-q)
#eXclude Root mode (-xr)
#print logs to current directory (-l)
#

#######################################################################################################################

#######################################################################################################################


import hashlib
import os
import sys
import shutil
import time
import re
import argparse
import getpass
import subprocess

#generate list of destination files of pairs of start and end files
def makeFileList(sourceList,dest,hashalg,hashlengths,excludeRoot):
    '''
returns a list of file(s) to hashsync
    '''
    flist = []
    for s in sourceList:
        if os.path.isdir(s) is False: #if first argument is a file it's p easy
            destinationObj = os.path.join(dest, os.path.basename(s)) # this is the file that we wanted to move, in its destination
            flist.extend([(s, destinationObj)])
            flist.extend([((s + "." + hashalg),(destinationObj + "." + hashalg))])
        #if the start object is a directory things get tricky
        elif os.path.isdir(s) is True:
            if excludeRoot: #this part makes the -xr flag work
                rootDirName = ""
            else:
                rootDirName = os.path.basename(s)
            if not s.endswith("/"):
                s = s + "/" #later, when we do some string subs, this keeps os.path.join() from breaking on a leading / I HATE HAVING TO DO THIS
            for dirs, subdirs, files in os.walk(s): #walk recursively through the dirtree
                for x in files: #ok, for all files rooted at start object
                    b = os.path.join(dirs,x) #make the full path to the file
                    b = b.replace(s,'') #extract just path relative to startObj (the subdirtree that x is in)
                    destinationObj = os.path.join(dest,rootDirName,b) #recombine the subdirtree with given destination (and file.extension)
                    sourceFile = os.path.join(dirs, x) #grab the start file full path
                    sourceFilename, ext = os.path.splitext(sourceFile) #separate extension from filename
                    sourceBasename = os.path.basename(sourceFilename)
                    if (not sourceBasename.startswith('.')) and (not ext.replace(".","") in hashlengths): #check that the file isn't DS_store and that file found doesn't have the hash extension (no hashes of hashes here my friend)
                        flist.extend([(sourceFile,destinationObj)]) #add these items as a tuple to the list of files
                        flist.extend([((sourceFile + "." + hashalg),(destinationObj + "." + hashalg))])
        else:
            print("Critical Error. Could not determine if the input is a file or directory. Something is very wrong.")
            sys.exit()
    return flist


#generate checksums for both source and dest
def generateHash(inputFile, hashalg, blocksize=65536):
    '''
    using a buffer, hash the file
    '''
    with open(inputFile, 'rb') as f:
        hasher = hashlib.new(hashalg) #grab the hashing algorithm decalred by user
        buf = f.read(blocksize) # read the file into a buffer cause it's more efficient for big files
        while len(buf) > 0: # little loop to keep reading
            hasher.update(buf) # here's where the hash is actually generated
            buf = f.read(blocksize) # keep reading
    return hasher.hexdigest()

#write hash to a file
def writeHash(inputFile, hashalg, logDir, resultsList):
    logNewLine("Writing New Sidecar Hash For " + os.path.basename(inputFile) + "........",logDir)
    try:
        hash = generateHash(inputFile, hashalg)
        hashFile = inputFile + "." + hashalg
        txt = open(hashFile, "w") #old school
        txt.write(hash + " *" + os.path.basename(inputFile))
        txt.close()
        logSameLine("Complete!",logDir)
        resultsList[1] += 1
        return resultsList
    except:
        logSameLine("ERROR!",logDir)
        resultsList[4] += 1
        return resultsList

#gets the hash from a sidecar text file
def readHash(hashFile, hashlength):
    with open(hashFile,'r') as f: #open it
        storedHash = re.search('\w{'+hashlength+'}',f.read()).group() #get the hash
    return storedHash

#verify's a hash from the sidecar file
def verifyHash(sidecarFile, hashalg, hashlength, logDir, resultsList):
    logNewLine("Verifying Hash For " + os.path.basename(sidecarFile) + "........",logDir)
    try:
        writtenHash = readHash(sidecarFile, hashlength) #read the hash written in the sidecar
        generatedHash = generateHash(sidecarFile.replace("." + hashalg, ""), hashalg) #generate the hash of the associated file
        if writtenHash == generatedHash: #then verify the checksums
            pass
            logSameLine("Complete! Checksum Verified",logDir)
            resultsList[2] += 1
            return resultsList
        else:
            pass
            logSameLine("ERROR: Checksum Verification Failed!",logDir)
            resultsList[4] += 1
            return resultsList
    except:
        logSameLine("ERROR!",logDir)

def removeUpToDateFiles(flist, hashalg, hashlength):
    '''
    iterates through the input list, creates output list
    '''
    removeFilesList = []
    reloadFileList = []
    for sf,df in flist: #get each item of flist, which contains a source file (sf) and destination file (df)
        if os.path.exists(df): #if the destination file exists
            if (hashalg in df): #this portion checks to see if a sidecar exists in the destination without an associated file.
                if not os.path.exists(sf): #if the destination checksum exists but the source doesn't then reload the hash. if we don't do this it can crash in some cases
                    reloadFileList.extend([(sf,df)])
                if not os.path.exists(df.replace("." + hashalg, "")):
                    reloadFileList.extend([(sf,df)]) #if this happens we need to reload the sidecar file so that it verifies when the assocaited file is reloaded
            elif (compare_modtime(sf,df)) and (compare_filesize(sf,df)): #if source files are older than destination and same size, things are good
                removeFilesList.extend([(sf,df)])
                if (hashalg not in sf): #this removes the associated sidecar files
                    removeFilesList.extend(((sf + "." + hashalg),(df + "." + hashalg)))
            else: #if destination files are older than source files, need to resync
                pass #leave item in the list, so it's reloaded
    flist = [x for x in flist if x not in removeFilesList] #remove the files already there from the file list
    flist.extend(reloadFileList)
    return flist



#performs the copy from one location (src) to another (dest)
def copyFiles(src,dest,logDir,resultsList):
    '''
    copies src file to the dest location
    using the native copy utility for a given platform
    '''
    logNewLine("Copying file: " + os.path.basename(src) + "........",logDir)
    try:
        win=mac=False
        if sys.platform.startswith("darwin"):
            mac=True
        elif sys.platform.startswith("win"):
            win=True
        cmd=None
        _dest = os.path.dirname(os.path.normpath(dest))
        if not os.path.exists(_dest):
            os.makedirs(_dest)
        if mac:
            cmd=['cp',src,dest]
        elif win:
            srce = os.path.dirname(src)
            dest = os.path.dirname(dest)
            name,ext = os.path.split(src)
            cmd=['robocopy',srce,dest,ext]
            print(cmd)

        subprocess.call(cmd)
        logSameLine("Complete!",logDir)
        resultsList[0] += 1
        return resultsList
    except:
        logSameLine("ERROR!",logDir)
        resultsList[4] += 1
        return resultsList

#this steps through the file list, creates checksums when needed, moves files, and verifies
def processList(flist, hashalg, hashlength, logDir, resultsList):
    for sf,df in flist:
        if (hashalg not in sf):    #if not a hash file
            if os.path.exists(sf + "." + hashalg): #see if hash file exists
                if compare_modtime(sf, sf + "." + hashalg): #check if hash file is newer than actual file
                    resultsList = copyFiles(sf,df,logDir, resultsList) #if hash is newer, we're good to move the file
                else:
                    logNewLine("WARNING! The checksum file for " + os.path.basename(sf) + " is out of date. Creating a new sidecare checksum file",logDir)
                    resultsList[3] += 1
                    resultsList = writeHash(sf, hashalg,logDir,resultsList) #if has is older, recreate sidecar file
                    resultsList = copyFiles(sf,df,logDir, resultsList) #once the hash is written we can move the file
            else:
                resultsList = writeHash(sf, hashalg,logDir, resultsList) #if no sidecar file exists, make one!
                resultsList = copyFiles(sf,df,logDir, resultsList) #once the hash is written we can move the file
        else: #if it's a hash file
            if os.path.exists(sf): #if the has exists
                resultsList = copyFiles(sf,df,logDir, resultsList) #move the hashe
                resultsList = verifyHash(df, hashalg, hashlength, logDir, resultsList) #verify the checksum from the sidecar file
            else: #if the hash file doesn't exist, we gotta make it, then copy it, then verify it (this is important if the file and hash are loaded, but the source hash has been deleted. could be skipped if i could figure this out)
                resultsList = writeHash(sf.replace("." + hashalg, ""), hashalg,logDir,resultsList) #write the hash
                resultsList = copyFiles(sf,df,logDir, resultsList) #move the hash
                resultsList = verifyHash(df, hashalg, hashlength, logDir, resultsList) #verify the checksum from the sidecar file
    return resultsList

#compares two files. If olderFile was created before newerFile then return true. else return false
def compare_modtime(olderFile, newerFile):
    olderFileMod = os.path.getmtime(olderFile)
    newerFileMod = os.path.getmtime(newerFile)
    if newerFileMod > olderFileMod:
        return True
    else:
        return False

#compares two files. If file1 and file2 are same size returns true, otherwise returns false
def compare_filesize(file1, file2):
    file1Size = os.stat(file1).st_size
    file2Size = os.stat(file2).st_size
    if file1Size == file2Size:
        return True
    else:
        return False

def initLog(sourceList,destination,hashalg):
    '''
    initializes log file
    '''
    txtFile = open(destination + "/LoadingScript.log", "a+")
    txtFile.write("Load and Verify Script Started at: " + time.strftime("%Y-%m-%d_%H:%M:%S") + "\n")
    for f in sourceList:
        txtFile.write("From: " + f + "\n")
    txtFile.write("To: " + destination + "\n")
    txtFile.write("Hash algorithm: " + hashalg + "\n")
    txtFile.write("\n\n")
    txtFile.close()

def logNewLine(text,destination):
    txtFile = open(destination + "/LoadingScript.log", "a+")
    txtFile.write("\n" + time.strftime("%Y-%m-%d_%H:%M:%S") + ": " + text)

def logSameLine(text,destination):
    txtFile = open(destination + "/LoadingScript.log", "a+")
    txtFile.write(text)

def make_args():
    '''
    initialize arguments from the cli
    '''
    parser = argparse.ArgumentParser()
    parser.add_argument('-d','--delete',action='store_true',dest='c',default=False,help="delete source files after copying")
    parser.add_argument('-q','--quiet',action='store_true',dest='q',default=False,help="quiet mode, don't print anything to console")
    parser.add_argument('-a','--algorithm',action='store',dest='a',default='md5',choices=['md5','sha1','sha256','sha512'],help="the hashing algorithm to use")
    parser.add_argument('sourceAndDestObj',nargs='+',help="the file or directory to hash/ move/ copy/ verify/ delete")
    parser.add_argument('-xr','--bottomFolder',action='store_true',dest='xr',default=False,help="eXclude Root mode. This will move the folders and directories in the selecte input folders directly in the output, rather than the standard rsync style nesting")
    #parser.add_argument('destinationDir',nargs='?',default=os.getcwd(),help="the destination parent directory")
    return parser.parse_args()

def main():
    '''
    do the thing
    '''
    #init args from cli
    args = make_args()
    ###INIT VARS###
    flist = []
    successCopyCount = 0
    successWriteHashCount = 0
    successVerifyHashCount = 0
    warningCount = 0
    errorCount =0
    resultsList = [successCopyCount,successWriteHashCount,successVerifyHashCount,warningCount,errorCount]
    ###END INIT###

    #housekeeping
    if len(args.sourceAndDestObj) < 2: #if less than two input arguments we have to exit
        print("CRITICAL ERROR: You must give this script at least two input arguments, with the last argument being the destination directory")
        sys.exit()

    destinationDir = args.sourceAndDestObj.pop(-1)

    if os.path.isdir(destinationDir) is False:
        print("CRITICAL ERROR: The last argument must be a directory")
        sys.exit()

    sourceList = args.sourceAndDestObj

    for s in sourceList:
        s.replace("\\","/") #everything is gonna break if we don't do this for windows ppl
    if args.q is True: #quiet mode redirects standard out to nul
        f = open(os.devnull,'w')
        sys.stdout = f
    hashAlgorithm = hashlib.new(args.a) #creates a hashlib object that is the algorithm we're using
    hashlengths = {'md5':'32','sha1':'40','sha256':'64','sha512':'128'}
    hashlength = hashlengths[args.a] #set value for comparison later

    #Initialize log
    initLog(sourceList,destinationDir,args.a)

    #Create giant list of input files and output files
    flist = makeFileList(sourceList,destinationDir,args.a,hashlengths,args.xr)

    #updates file list, removing all files that were already there
    flist = removeUpToDateFiles(flist, args.a, hashlength)

    #process files in the list
    resultsList = processList(flist, args.a, hashlength, destinationDir, resultsList)

    print("\nCopy and Verify Completed! Here are the results. See log for details")
    print("\nFiles Copied: " + str(resultsList[0]))
    print("Checksums Written: " + str(resultsList[1]))
    print("Checksums Verified: " + str(resultsList[2]))
    print("Warnings: " + str(resultsList[3]))
    print("Errors: " + str(resultsList[4]))
    print("\n")

    logSameLine("\n***************************\n\n\n\n",destinationDir)

main()