Atlas/Atlas-Install
1
0
Fork 0
mirror of https://github.com/Faraphel/Atlas-Install.git synced 2025-07-03 11:18:26 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixed track download and sha1 check, track should now be downloaded faster and in parallel

Browse source
This commit is contained in:
raphael60650 2021-07-22 22:35:41 +02:00
parent 53ee294915
commit 4ac7ac3fe5
3 changed files with 54 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
source/Game.py
View file

@ -357,17 +357,11 @@ class Game:
def patch_tracks(self):
max_process = self.gui.intvar_process_track.get()
process_list = {}
thread_list = {}
error_count, error_max = 0, 3
def add_process(track):
nonlocal error_count, error_max, process_list
track_file = track.get_track_name()
total_track = len(self.ctconfig.all_tracks)
process_list[track_file] = None # Used for showing track in progress even if there's no process
self.gui.progress(statut=self.gui.translate("Converting tracks", f"\n({i + 1}/{total_track})\n",
"\n".join(process_list.keys())), add=1)
nonlocal error_count, error_max, thread_list
for _track in [track.file_szs, track.file_wu8]:
if os.path.exists(_track):
@ -394,7 +388,7 @@ class Game:
if track.sha1:
if not self.gui.boolvar_dont_check_track_sha1.get():
if not track.check_sha1(): # Check si le sha1 du fichier est le bon
if track.check_sha1() != 0: # if track sha1 is not the one excepted
error_count += 1
if error_count > error_max: # Too much track wasn't correctly converted
messagebox.showerror(
@ -408,7 +402,7 @@ class Game:
if not (os.path.exists(track.file_szs)) or download_returncode == 3:
# returncode 3 is track has been updated
if os.path.exists(track.file_wu8):
process_list[track_file] = track.convert_wu8_to_szs()
track.convert_wu8_to_szs()
else:
messagebox.showerror(self.gui.translate("Error"),
self.gui.translate("Can't convert track.\nEnable track download and retry."))
@ -418,58 +412,46 @@ class Game:
return 0
def clean_process():
nonlocal error_count, error_max, process_list
nonlocal error_count, error_max, thread_list
for track_file, process in process_list.copy().items():
if process is not None:
if process.poll() is None:
pass # if the process is still running
else: # process ended
process_list.pop(track_file)
stderr = process.stderr.read()
if b"wszst: ERROR" in stderr: # Error occured
os.remove(track.file_szs)
error_count += 1
if error_count > error_max: # Too much track wasn't correctly converted
messagebox.showerror(
self.gui.translate("Error"),
self.gui.translate("Too much track had a conversion issue."))
raise CantConvertTrack()
else: # if the error max hasn't been reach
messagebox.showwarning(
self.gui.translate("Warning"),
self.gui.translate("The track", " ", track.file_wu8,
"do not have been properly converted.",
f" ({error_count} / {error_max})"))
else:
if self.gui.boolvar_del_track_after_conv.get(): os.remove(track.file_wu8)
else:
process_list.pop(track_file)
if not (any(process_list.values())): return 1 # if there is no more process
for track_key, thread in thread_list.copy().items():
if not thread.is_alive(): # if conversion ended
thread_list.pop(track_key)
"""stderr = thread.stderr.read()
if b"wszst: ERROR" in stderr: # Error occured
os.remove(track.file_szs)
error_count += 1
if error_count > error_max: # Too much track wasn't correctly converted
messagebox.showerror(
self.gui.translate("Error"),
self.gui.translate("Too much track had a conversion issue."))
raise CantConvertTrack()
else: # if the error max hasn't been reach
messagebox.showwarning(
self.gui.translate("Warning"),
self.gui.translate("The track", " ", track.file_wu8,
"do not have been properly converted.",
f" ({error_count} / {error_max})"))
else:
if self.gui.boolvar_del_track_after_conv.get(): os.remove(track.file_wu8)"""
if not (any(thread_list.values())): return 1 # if there is no more process
if len(process_list):
return 1
else:
return 0
if len(thread_list): return 1
else: return 0
total_track = len(self.ctconfig.all_tracks)
for i, track in enumerate(self.ctconfig.all_tracks):
while True:
if len(process_list) < max_process:
returncode = add_process(track)
if returncode == 0:
break
elif returncode == -1:
return -1 # if error occur, stop function
elif clean_process() == -1:
return -1
if len(thread_list) < max_process:
thread_list[track.file_wu8] = Thread(target=add_process, args=[track])
thread_list[track.file_wu8].setDaemon(True)
thread_list[track.file_wu8].start()
self.gui.progress(statut=self.gui.translate("Converting tracks", f"\n({i + 1}/{total_track})\n",
"\n".join(thread_list.keys())), add=1)
break
clean_process()
while True:
returncode = clean_process()
if returncode == 1:
break # End the process if all process ended
elif returncode == 0:
pass
else:
return -1
while clean_process() != 1:
pass # End the process if all process ended
return 0

16
source/Track.py
View file

@ -23,6 +23,8 @@ class Track:
self.score = score # Track score between 1 and 3 stars
self.warning = warning # Track bug level (1 = minor, 2 = major)
self.note = note # Note about the track
self.track_wu8_dir = track_wu8_dir
self.track_szs_dir = track_szs_dir
self.file_wu8 = f"{track_wu8_dir}/{self.get_track_name()}.wu8"
self.file_szs = f"{track_szs_dir}/{self.get_track_name()}.szs"
@ -30,13 +32,12 @@ class Track:
return f"{self.get_track_name()} sha1={self.sha1} score={self.score}"
def check_sha1(self):
if wszst.sha1(self.file_wu8) == self.sha1:
return 0
else:
return -1
ws = wszst.sha1(self.file_wu8)
if wszst.sha1(self.file_wu8) == self.sha1: return 0
else: return -1
def convert_wu8_to_szs(self):
return wszst.normalize(src_file=self.file_wu8, use_popen=True)
return wszst.normalize(src_file=self.file_wu8)
def download_wu8(self, github_content_root: str):
returncode = 0
@ -50,7 +51,7 @@ class Track:
if dl.status_code == 200: # if page is found
with open(self.file_wu8, "wb") as file:
chunk_size = 4096
chunk_size = 524288
for i, chunk in enumerate(dl.iter_content(chunk_size=chunk_size)):
file.write(chunk)
file.flush()
@ -121,3 +122,6 @@ class Track:
def load_from_json(self, track_json: dict):
for key, value in track_json.items(): # load all value in the json as class attribute
setattr(self, key, value)
self.file_wu8 = f"{self.track_wu8_dir}/{self.get_track_name()}.wu8"
self.file_szs = f"{self.track_szs_dir}/{self.get_track_name()}.szs"

12
source/wszst.py
View file

@ -2,18 +2,19 @@ from .definition import *
import subprocess
def sha1(file):
def sha1(file, autoadd_path: str = "./file/auto-add/"):
"""
:param autoadd_path:
:param file: track file to check sha1
:return: track's sha1
"""
return subprocess.run(["./tools/szs/wszst", "SHA1", file, "--autoadd-path", "./file/auto-add/"],
return subprocess.run(["./tools/szs/wszst", "SHA1", file, "--autoadd-path", autoadd_path],
check=True, creationflags=CREATE_NO_WINDOW,
stdout=subprocess.PIPE).stdout.decode().split(" ")[0]
def normalize(src_file: str, dest_dir: str = "./file/Track/", dest_name: str = "%N.szs",
output_format: str = "szs", autoadd_path: str = "./file/auto-add/", use_popen: bool = False):
output_format: str = "szs", autoadd_path: str = "./file/auto-add/"):
"""
:param use_popen: True if you want to use Popen to convert
:param src_file: source file
@ -23,10 +24,7 @@ def normalize(src_file: str, dest_dir: str = "./file/Track/", dest_name: str = "
:param autoadd_path: path of the auto-add directory
:return: 0
"""
if use_popen: cmd_process = subprocess.Popen
else: cmd_process = subprocess.run
return cmd_process([
"./tools/szs/wszst", "NORMALIZE", src_file, "--DEST",
subprocess.run(["./tools/szs/wszst", "NORMALIZE", src_file, "--DEST",
dest_dir+dest_name, "--"+output_format, "--overwrite", "--autoadd-path",
autoadd_path], creationflags=CREATE_NO_WINDOW, stderr=subprocess.PIPE)