Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

ThreadFileDownloader.py 2.7 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
  1. import io
  2. import threading
  3. import urllib.request
  4. from pathlib import Path
  5. from typing import Union
  6. class ThreadFileDownloader:
  7. """
  8. FileDownloader using sub thread
  9. url str
  10. savepath(None) str,Path
  11. Use .get_error() to check the error
  12. """
  13. def __init__(self, url, savepath : Union[str, Path] = None):
  14. if savepath is not None:
  15. savepath = Path(savepath)
  16. self._partpath = savepath.parent / ( savepath.name + '.part' )
  17. else:
  18. self._partpath = None
  19. self._savepath = savepath
  20. self._url = url
  21. self._error = None
  22. self._file_size = None
  23. self._file_size_dl = None
  24. self._bytes = None
  25. threading.Thread(target=self._thread, daemon=True).start()
  26. def get_progress(self) -> float:
  27. """
  28. return progress of downloading as [0.0...100.0] value
  29. where 100.0 mean download is completed
  30. """
  31. if self._file_size is None or self._file_size_dl is None:
  32. return 0.0
  33. return (self._file_size_dl / self._file_size) * 100.0
  34. def get_bytes(self) -> bytes:
  35. """
  36. return bytes of downloaded file if savepath is not defined
  37. """
  38. return self._bytes
  39. def get_error(self) -> Union[str, None]:
  40. """
  41. returns error string or None if no error
  42. """
  43. return self._error
  44. def _thread(self):
  45. try:
  46. url_req = urllib.request.urlopen(self._url)
  47. file_size = self._file_size = int( url_req.getheader('content-length') )
  48. self._file_size_dl = 0
  49. savepath = self._savepath
  50. partpath = self._partpath
  51. if partpath is not None:
  52. if partpath.exists():
  53. partpath.unlink()
  54. f = open(partpath, 'wb')
  55. else:
  56. f = io.BytesIO()
  57. while url_req is not None:
  58. buffer = url_req.read(8192)
  59. if not buffer:
  60. break
  61. f.write(buffer)
  62. new_file_size_dl = self._file_size_dl + len(buffer)
  63. if new_file_size_dl >= file_size:
  64. if partpath is not None:
  65. f.close()
  66. if savepath.exists():
  67. savepath.unlink()
  68. partpath.rename(savepath)
  69. else:
  70. self._bytes = f.getvalue()
  71. f.close()
  72. url_req.close()
  73. url_req = None
  74. self._file_size_dl = new_file_size_dl
  75. except Exception as e:
  76. self._error = str(e)
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...