mirror of
https://github.com/Faraphel/Atlas-Install.git
synced 2025-07-03 11:18:26 +02:00
fixed track download and sha1 check, track should now be downloaded faster and in parallel
This commit is contained in:
parent
53ee294915
commit
4ac7ac3fe5
3 changed files with 54 additions and 70 deletions
|
|
@ -357,17 +357,11 @@ class Game:
|
|||
|
||||
def patch_tracks(self):
|
||||
max_process = self.gui.intvar_process_track.get()
|
||||
process_list = {}
|
||||
thread_list = {}
|
||||
error_count, error_max = 0, 3
|
||||
|
||||
def add_process(track):
|
||||
nonlocal error_count, error_max, process_list
|
||||
track_file = track.get_track_name()
|
||||
total_track = len(self.ctconfig.all_tracks)
|
||||
|
||||
process_list[track_file] = None # Used for showing track in progress even if there's no process
|
||||
self.gui.progress(statut=self.gui.translate("Converting tracks", f"\n({i + 1}/{total_track})\n",
|
||||
"\n".join(process_list.keys())), add=1)
|
||||
nonlocal error_count, error_max, thread_list
|
||||
|
||||
for _track in [track.file_szs, track.file_wu8]:
|
||||
if os.path.exists(_track):
|
||||
|
|
@ -394,7 +388,7 @@ class Game:
|
|||
|
||||
if track.sha1:
|
||||
if not self.gui.boolvar_dont_check_track_sha1.get():
|
||||
if not track.check_sha1(): # Check si le sha1 du fichier est le bon
|
||||
if track.check_sha1() != 0: # if track sha1 is not the one excepted
|
||||
error_count += 1
|
||||
if error_count > error_max: # Too much track wasn't correctly converted
|
||||
messagebox.showerror(
|
||||
|
|
@ -408,7 +402,7 @@ class Game:
|
|||
if not (os.path.exists(track.file_szs)) or download_returncode == 3:
|
||||
# returncode 3 is track has been updated
|
||||
if os.path.exists(track.file_wu8):
|
||||
process_list[track_file] = track.convert_wu8_to_szs()
|
||||
track.convert_wu8_to_szs()
|
||||
else:
|
||||
messagebox.showerror(self.gui.translate("Error"),
|
||||
self.gui.translate("Can't convert track.\nEnable track download and retry."))
|
||||
|
|
@ -418,15 +412,12 @@ class Game:
|
|||
return 0
|
||||
|
||||
def clean_process():
|
||||
nonlocal error_count, error_max, process_list
|
||||
nonlocal error_count, error_max, thread_list
|
||||
|
||||
for track_file, process in process_list.copy().items():
|
||||
if process is not None:
|
||||
if process.poll() is None:
|
||||
pass # if the process is still running
|
||||
else: # process ended
|
||||
process_list.pop(track_file)
|
||||
stderr = process.stderr.read()
|
||||
for track_key, thread in thread_list.copy().items():
|
||||
if not thread.is_alive(): # if conversion ended
|
||||
thread_list.pop(track_key)
|
||||
"""stderr = thread.stderr.read()
|
||||
if b"wszst: ERROR" in stderr: # Error occured
|
||||
os.remove(track.file_szs)
|
||||
error_count += 1
|
||||
|
|
@ -442,34 +433,25 @@ class Game:
|
|||
"do not have been properly converted.",
|
||||
f" ({error_count} / {error_max})"))
|
||||
else:
|
||||
if self.gui.boolvar_del_track_after_conv.get(): os.remove(track.file_wu8)
|
||||
else:
|
||||
process_list.pop(track_file)
|
||||
if not (any(process_list.values())): return 1 # if there is no more process
|
||||
if self.gui.boolvar_del_track_after_conv.get(): os.remove(track.file_wu8)"""
|
||||
if not (any(thread_list.values())): return 1 # if there is no more process
|
||||
|
||||
if len(process_list):
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
if len(thread_list): return 1
|
||||
else: return 0
|
||||
|
||||
total_track = len(self.ctconfig.all_tracks)
|
||||
for i, track in enumerate(self.ctconfig.all_tracks):
|
||||
while True:
|
||||
if len(process_list) < max_process:
|
||||
returncode = add_process(track)
|
||||
if returncode == 0:
|
||||
if len(thread_list) < max_process:
|
||||
thread_list[track.file_wu8] = Thread(target=add_process, args=[track])
|
||||
thread_list[track.file_wu8].setDaemon(True)
|
||||
thread_list[track.file_wu8].start()
|
||||
self.gui.progress(statut=self.gui.translate("Converting tracks", f"\n({i + 1}/{total_track})\n",
|
||||
"\n".join(thread_list.keys())), add=1)
|
||||
break
|
||||
elif returncode == -1:
|
||||
return -1 # if error occur, stop function
|
||||
elif clean_process() == -1:
|
||||
return -1
|
||||
clean_process()
|
||||
|
||||
while True:
|
||||
returncode = clean_process()
|
||||
if returncode == 1:
|
||||
break # End the process if all process ended
|
||||
elif returncode == 0:
|
||||
pass
|
||||
else:
|
||||
return -1
|
||||
while clean_process() != 1:
|
||||
pass # End the process if all process ended
|
||||
|
||||
return 0
|
||||
|
|
@ -23,6 +23,8 @@ class Track:
|
|||
self.score = score # Track score between 1 and 3 stars
|
||||
self.warning = warning # Track bug level (1 = minor, 2 = major)
|
||||
self.note = note # Note about the track
|
||||
self.track_wu8_dir = track_wu8_dir
|
||||
self.track_szs_dir = track_szs_dir
|
||||
self.file_wu8 = f"{track_wu8_dir}/{self.get_track_name()}.wu8"
|
||||
self.file_szs = f"{track_szs_dir}/{self.get_track_name()}.szs"
|
||||
|
||||
|
|
@ -30,13 +32,12 @@ class Track:
|
|||
return f"{self.get_track_name()} sha1={self.sha1} score={self.score}"
|
||||
|
||||
def check_sha1(self):
|
||||
if wszst.sha1(self.file_wu8) == self.sha1:
|
||||
return 0
|
||||
else:
|
||||
return -1
|
||||
ws = wszst.sha1(self.file_wu8)
|
||||
if wszst.sha1(self.file_wu8) == self.sha1: return 0
|
||||
else: return -1
|
||||
|
||||
def convert_wu8_to_szs(self):
|
||||
return wszst.normalize(src_file=self.file_wu8, use_popen=True)
|
||||
return wszst.normalize(src_file=self.file_wu8)
|
||||
|
||||
def download_wu8(self, github_content_root: str):
|
||||
returncode = 0
|
||||
|
|
@ -50,7 +51,7 @@ class Track:
|
|||
|
||||
if dl.status_code == 200: # if page is found
|
||||
with open(self.file_wu8, "wb") as file:
|
||||
chunk_size = 4096
|
||||
chunk_size = 524288
|
||||
for i, chunk in enumerate(dl.iter_content(chunk_size=chunk_size)):
|
||||
file.write(chunk)
|
||||
file.flush()
|
||||
|
|
@ -121,3 +122,6 @@ class Track:
|
|||
def load_from_json(self, track_json: dict):
|
||||
for key, value in track_json.items(): # load all value in the json as class attribute
|
||||
setattr(self, key, value)
|
||||
|
||||
self.file_wu8 = f"{self.track_wu8_dir}/{self.get_track_name()}.wu8"
|
||||
self.file_szs = f"{self.track_szs_dir}/{self.get_track_name()}.szs"
|
||||
|
|
|
|||
|
|
@ -2,18 +2,19 @@ from .definition import *
|
|||
import subprocess
|
||||
|
||||
|
||||
def sha1(file):
|
||||
def sha1(file, autoadd_path: str = "./file/auto-add/"):
|
||||
"""
|
||||
:param autoadd_path:
|
||||
:param file: track file to check sha1
|
||||
:return: track's sha1
|
||||
"""
|
||||
return subprocess.run(["./tools/szs/wszst", "SHA1", file, "--autoadd-path", "./file/auto-add/"],
|
||||
return subprocess.run(["./tools/szs/wszst", "SHA1", file, "--autoadd-path", autoadd_path],
|
||||
check=True, creationflags=CREATE_NO_WINDOW,
|
||||
stdout=subprocess.PIPE).stdout.decode().split(" ")[0]
|
||||
|
||||
|
||||
def normalize(src_file: str, dest_dir: str = "./file/Track/", dest_name: str = "%N.szs",
|
||||
output_format: str = "szs", autoadd_path: str = "./file/auto-add/", use_popen: bool = False):
|
||||
output_format: str = "szs", autoadd_path: str = "./file/auto-add/"):
|
||||
"""
|
||||
:param use_popen: True if you want to use Popen to convert
|
||||
:param src_file: source file
|
||||
|
|
@ -23,10 +24,7 @@ def normalize(src_file: str, dest_dir: str = "./file/Track/", dest_name: str = "
|
|||
:param autoadd_path: path of the auto-add directory
|
||||
:return: 0
|
||||
"""
|
||||
if use_popen: cmd_process = subprocess.Popen
|
||||
else: cmd_process = subprocess.run
|
||||
return cmd_process([
|
||||
"./tools/szs/wszst", "NORMALIZE", src_file, "--DEST",
|
||||
subprocess.run(["./tools/szs/wszst", "NORMALIZE", src_file, "--DEST",
|
||||
dest_dir+dest_name, "--"+output_format, "--overwrite", "--autoadd-path",
|
||||
autoadd_path], creationflags=CREATE_NO_WINDOW, stderr=subprocess.PIPE)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue