tdm
/
lineage-updater
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve object hash performance and other Various fixes 

* Hash filename, timestamp, and size for obect ID.
 * Coerce all args to lowercase.
 * Ignore stat() failures and continue processing.
 * Truncate cache file when writing.
 * Close cache file after writing.
 * Catch most exceptions as Exception.
This commit is contained in:
Tom Marshall 2022-09-02 09:52:47 -07:00
parent c4ed89e4f2
commit 6c7cd369fa
1 changed files with 18 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
lineage-updater
View File

@ -58,18 +58,10 @@ def args_parse():
arg = values.get(k, [])
if len(arg) != 1:
raise RuntimeError("No %s specified" % (k))
args[k] = arg[0]
args[k] = arg[0].lower()
return args
def file_hash(pathname):
f = open(pathname, 'rb')
buf = f.read()
f.close()
hasher = hashlib.sha1()
hasher.update(buf)
return hasher.hexdigest()
def filename_properties(pathname):
filename = os.path.basename(pathname)
if not filename.endswith('.zip'):
@ -111,21 +103,25 @@ def cache_refresh(cache):
pathname = "%s/%s" % (root, filename)
try:
props = filename_properties(pathname)
except ValueError:
st = os.stat(pathname)
except Exception:
continue
st = os.stat(pathname)
if (not pathname in cache or
st.st_size != cache[pathname]['size'] or
int(st.st_mtime) != cache[pathname]['datetime']):
hash = file_hash(pathname)
obj = dict()
obj['datetime'] = int(st.st_mtime)
obj['filename'] = filename
obj['id'] = hash
obj['romtype'] = props['buildtype']
obj['size'] = st.st_size
obj['url'] = "%s%s/%s" % (config['baseurl'], relpath, filename)
obj['version'] = props['version']
str = "%s.%d.%d" % (obj['filename'], obj['datetime'], obj['size'])
hasher = hashlib.sha1()
hasher.update(str.encode())
obj['id'] = hasher.hexdigest()
cache[pathname] = obj
stale = True
@ -150,8 +146,8 @@ def find_roms(device=None, buildtype=None, incremental=None):
update_cache = False
try:
cache = json.load(cf)
except:
dbg("Failed to read cache file")
except Exception as e:
dbg("Failed to read cache file: %s" % (e))
update_cache = True
if cf is None or ct < time.time() - 60:
@ -162,16 +158,19 @@ def find_roms(device=None, buildtype=None, incremental=None):
cf.seek(0, 0)
try:
json.dump(cache, cf)
cf.truncate()
except:
pass
fcntl.lockf(cf, fcntl.LOCK_UN)
cf.close()
cf = None
roms = []
for k, v in cache.items():
try:
props = filename_properties(k)
except ValueError:
dbg("Failed to parse cache filename")
except Exception as e:
dbg("Failed to parse cache filename: %s" % (e))
continue
if device and props['device'] != device:
@ -210,12 +209,12 @@ config = dict()
try:
config_load("%s/.lineageupdaterrc" % (home))
except RuntimeError as e:
except Exception as e:
die("Failed to load configuration: %s" % (str(e)))
try:
args = args_parse()
except RuntimeError as e:
except Exception as e:
die("Failed to parse arguments: %s\n" % (str(e)))
if cgi_method() != "GET":