Atlas/Atlas-Install
1
0
Fork 0
mirror of https://github.com/Faraphel/Atlas-Install.git synced 2025-07-03 19:28:25 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixed track download and sha1 check, track should now be downloaded faster and in parallel

Browse source
This commit is contained in:
raphael60650 2021-07-22 22:35:41 +02:00
parent 53ee294915
commit 4ac7ac3fe5
3 changed files with 54 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
source/Game.py
View file

@ -357,17 +357,11 @@ class Game:
def patch_tracks(self): def patch_tracks(self):
max_process = self.gui.intvar_process_track.get() max_process = self.gui.intvar_process_track.get()
process_list = {} thread_list = {}
error_count, error_max = 0, 3 error_count, error_max = 0, 3
def add_process(track): def add_process(track):
nonlocal error_count, error_max, process_list nonlocal error_count, error_max, thread_list
track_file = track.get_track_name()
total_track = len(self.ctconfig.all_tracks)
process_list[track_file] = None # Used for showing track in progress even if there's no process
self.gui.progress(statut=self.gui.translate("Converting tracks", f"\n({i + 1}/{total_track})\n",
"\n".join(process_list.keys())), add=1)
for _track in [track.file_szs, track.file_wu8]: for _track in [track.file_szs, track.file_wu8]:
if os.path.exists(_track): if os.path.exists(_track):
@ -394,7 +388,7 @@ class Game:
if track.sha1: if track.sha1:
if not self.gui.boolvar_dont_check_track_sha1.get(): if not self.gui.boolvar_dont_check_track_sha1.get():
if not track.check_sha1(): # Check si le sha1 du fichier est le bon if track.check_sha1() != 0: # if track sha1 is not the one excepted
error_count += 1 error_count += 1
if error_count > error_max: # Too much track wasn't correctly converted if error_count > error_max: # Too much track wasn't correctly converted
messagebox.showerror( messagebox.showerror(
@ -408,7 +402,7 @@ class Game:
if not (os.path.exists(track.file_szs)) or download_returncode == 3: if not (os.path.exists(track.file_szs)) or download_returncode == 3:
# returncode 3 is track has been updated # returncode 3 is track has been updated
if os.path.exists(track.file_wu8): if os.path.exists(track.file_wu8):
process_list[track_file] = track.convert_wu8_to_szs() track.convert_wu8_to_szs()
else: else:
messagebox.showerror(self.gui.translate("Error"), messagebox.showerror(self.gui.translate("Error"),
self.gui.translate("Can't convert track.\nEnable track download and retry.")) self.gui.translate("Can't convert track.\nEnable track download and retry."))
@ -418,58 +412,46 @@ class Game:
return 0 return 0
def clean_process(): def clean_process():
nonlocal error_count, error_max, process_list nonlocal error_count, error_max, thread_list
for track_file, process in process_list.copy().items(): for track_key, thread in thread_list.copy().items():
if process is not None: if not thread.is_alive(): # if conversion ended
if process.poll() is None: thread_list.pop(track_key)
pass # if the process is still running """stderr = thread.stderr.read()
else: # process ended if b"wszst: ERROR" in stderr: # Error occured
process_list.pop(track_file) os.remove(track.file_szs)
stderr = process.stderr.read() error_count += 1
if b"wszst: ERROR" in stderr: # Error occured if error_count > error_max: # Too much track wasn't correctly converted
os.remove(track.file_szs) messagebox.showerror(
error_count += 1 self.gui.translate("Error"),
if error_count > error_max: # Too much track wasn't correctly converted self.gui.translate("Too much track had a conversion issue."))
messagebox.showerror( raise CantConvertTrack()
self.gui.translate("Error"), else: # if the error max hasn't been reach
self.gui.translate("Too much track had a conversion issue.")) messagebox.showwarning(
raise CantConvertTrack() self.gui.translate("Warning"),
else: # if the error max hasn't been reach self.gui.translate("The track", " ", track.file_wu8,
messagebox.showwarning( "do not have been properly converted.",
self.gui.translate("Warning"), f" ({error_count} / {error_max})"))
self.gui.translate("The track", " ", track.file_wu8, else:
"do not have been properly converted.", if self.gui.boolvar_del_track_after_conv.get(): os.remove(track.file_wu8)"""
f" ({error_count} / {error_max})")) if not (any(thread_list.values())): return 1 # if there is no more process
else:
if self.gui.boolvar_del_track_after_conv.get(): os.remove(track.file_wu8)
else:
process_list.pop(track_file)
if not (any(process_list.values())): return 1 # if there is no more process
if len(process_list): if len(thread_list): return 1
return 1 else: return 0
else:
return 0
total_track = len(self.ctconfig.all_tracks)
for i, track in enumerate(self.ctconfig.all_tracks): for i, track in enumerate(self.ctconfig.all_tracks):
while True: while True:
if len(process_list) < max_process: if len(thread_list) < max_process:
returncode = add_process(track) thread_list[track.file_wu8] = Thread(target=add_process, args=[track])
if returncode == 0: thread_list[track.file_wu8].setDaemon(True)
break thread_list[track.file_wu8].start()
elif returncode == -1: self.gui.progress(statut=self.gui.translate("Converting tracks", f"\n({i + 1}/{total_track})\n",
return -1 # if error occur, stop function "\n".join(thread_list.keys())), add=1)
elif clean_process() == -1: break
return -1 clean_process()
while True: while clean_process() != 1:
returncode = clean_process() pass # End the process if all process ended
if returncode == 1:
break # End the process if all process ended
elif returncode == 0:
pass
else:
return -1
return 0 return 0

16
source/Track.py
View file

@ -23,6 +23,8 @@ class Track:
self.score = score # Track score between 1 and 3 stars self.score = score # Track score between 1 and 3 stars
self.warning = warning # Track bug level (1 = minor, 2 = major) self.warning = warning # Track bug level (1 = minor, 2 = major)
self.note = note # Note about the track self.note = note # Note about the track
self.track_wu8_dir = track_wu8_dir
self.track_szs_dir = track_szs_dir
self.file_wu8 = f"{track_wu8_dir}/{self.get_track_name()}.wu8" self.file_wu8 = f"{track_wu8_dir}/{self.get_track_name()}.wu8"
self.file_szs = f"{track_szs_dir}/{self.get_track_name()}.szs" self.file_szs = f"{track_szs_dir}/{self.get_track_name()}.szs"
@ -30,13 +32,12 @@ class Track:
return f"{self.get_track_name()} sha1={self.sha1} score={self.score}" return f"{self.get_track_name()} sha1={self.sha1} score={self.score}"
def check_sha1(self): def check_sha1(self):
if wszst.sha1(self.file_wu8) == self.sha1: ws = wszst.sha1(self.file_wu8)
return 0 if wszst.sha1(self.file_wu8) == self.sha1: return 0
else: else: return -1
return -1
def convert_wu8_to_szs(self): def convert_wu8_to_szs(self):
return wszst.normalize(src_file=self.file_wu8, use_popen=True) return wszst.normalize(src_file=self.file_wu8)
def download_wu8(self, github_content_root: str): def download_wu8(self, github_content_root: str):
returncode = 0 returncode = 0
@ -50,7 +51,7 @@ class Track:
if dl.status_code == 200: # if page is found if dl.status_code == 200: # if page is found
with open(self.file_wu8, "wb") as file: with open(self.file_wu8, "wb") as file:
chunk_size = 4096 chunk_size = 524288
for i, chunk in enumerate(dl.iter_content(chunk_size=chunk_size)): for i, chunk in enumerate(dl.iter_content(chunk_size=chunk_size)):
file.write(chunk) file.write(chunk)
file.flush() file.flush()
@ -121,3 +122,6 @@ class Track:
def load_from_json(self, track_json: dict): def load_from_json(self, track_json: dict):
for key, value in track_json.items(): # load all value in the json as class attribute for key, value in track_json.items(): # load all value in the json as class attribute
setattr(self, key, value) setattr(self, key, value)
self.file_wu8 = f"{self.track_wu8_dir}/{self.get_track_name()}.wu8"
self.file_szs = f"{self.track_szs_dir}/{self.get_track_name()}.szs"

12
source/wszst.py
View file

@ -2,18 +2,19 @@ from .definition import *
import subprocess import subprocess
def sha1(file): def sha1(file, autoadd_path: str = "./file/auto-add/"):
""" """
:param autoadd_path:
:param file: track file to check sha1 :param file: track file to check sha1
:return: track's sha1 :return: track's sha1
""" """
return subprocess.run(["./tools/szs/wszst", "SHA1", file, "--autoadd-path", "./file/auto-add/"], return subprocess.run(["./tools/szs/wszst", "SHA1", file, "--autoadd-path", autoadd_path],
check=True, creationflags=CREATE_NO_WINDOW, check=True, creationflags=CREATE_NO_WINDOW,
stdout=subprocess.PIPE).stdout.decode().split(" ")[0] stdout=subprocess.PIPE).stdout.decode().split(" ")[0]
def normalize(src_file: str, dest_dir: str = "./file/Track/", dest_name: str = "%N.szs", def normalize(src_file: str, dest_dir: str = "./file/Track/", dest_name: str = "%N.szs",
output_format: str = "szs", autoadd_path: str = "./file/auto-add/", use_popen: bool = False): output_format: str = "szs", autoadd_path: str = "./file/auto-add/"):
""" """
:param use_popen: True if you want to use Popen to convert :param use_popen: True if you want to use Popen to convert
:param src_file: source file :param src_file: source file
@ -23,10 +24,7 @@ def normalize(src_file: str, dest_dir: str = "./file/Track/", dest_name: str = "
:param autoadd_path: path of the auto-add directory :param autoadd_path: path of the auto-add directory
:return: 0 :return: 0
""" """
if use_popen: cmd_process = subprocess.Popen subprocess.run(["./tools/szs/wszst", "NORMALIZE", src_file, "--DEST",
else: cmd_process = subprocess.run
return cmd_process([
"./tools/szs/wszst", "NORMALIZE", src_file, "--DEST",
dest_dir+dest_name, "--"+output_format, "--overwrite", "--autoadd-path", dest_dir+dest_name, "--"+output_format, "--overwrite", "--autoadd-path",
autoadd_path], creationflags=CREATE_NO_WINDOW, stderr=subprocess.PIPE) autoadd_path], creationflags=CREATE_NO_WINDOW, stderr=subprocess.PIPE)