Register
Login
Resources
Docs Blog Datasets Glossary Case Studies Tutorials & Webinars
Product
Data Engine LLMs Platform Enterprise
Pricing Explore
Connect to our Discord channel

util.py 4.0 KB

You have to be logged in to leave a comment. Sign In
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
  1. import errno
  2. import json
  3. import logging
  4. import os
  5. from six.moves import queue
  6. import requests
  7. import subprocess
  8. import threading
  9. import time
  10. logger = logging.getLogger(__name__)
  11. # TODO: get rid of this
  12. try:
  13. import numpy as np
  14. except ImportError:
  15. pass
  16. class WandBJSONEncoder(json.JSONEncoder):
  17. """A JSON Encoder that handles some extra types."""
  18. def default(self, obj):
  19. # TODO: Some of this is just guessing. Be smarter.
  20. if isinstance(obj, np.ndarray):
  21. return obj.tolist()
  22. if isinstance(obj, np.generic):
  23. return np.asscalar(obj)
  24. if isinstance(obj, bytes):
  25. return obj.decode('utf-8')
  26. return json.JSONEncoder.default(self, obj)
  27. def json_dumps_safer(obj, **kwargs):
  28. """Convert obj to json, with some extra encodable types."""
  29. return json.dumps(obj, cls=WandBJSONEncoder, **kwargs)
  30. def make_json_if_not_number(v):
  31. """If v is not a basic type convert it to json."""
  32. if isinstance(v, (float, int)):
  33. return v
  34. return json_dumps_safer(v)
  35. def mkdir_exists_ok(path):
  36. try:
  37. os.makedirs(path)
  38. except OSError as exc:
  39. if exc.errno == errno.EEXIST and os.path.isdir(path):
  40. pass
  41. else:
  42. raise
  43. class SafeSubprocess(object):
  44. def __init__(self, args, env=None):
  45. self._args = args
  46. self._env = env
  47. self._stdout = queue.Queue()
  48. self._stderr = queue.Queue()
  49. self._popen = None
  50. self._stdout_thread = None
  51. self._stderr_thread = None
  52. def run(self):
  53. self._popen = subprocess.Popen(
  54. self._args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self._env)
  55. self._stdout_thread = self._spawn_reader_thread(
  56. self._popen.stdout, self._stdout)
  57. self._stderr_thread = self._spawn_reader_thread(
  58. self._popen.stderr, self._stderr)
  59. def _spawn_reader_thread(self, filelike, out_queue):
  60. def _reader_thread(filelike, out_queue):
  61. while True:
  62. out = filelike.read(64).decode('utf-8')
  63. if not out:
  64. break
  65. out_queue.put(out)
  66. threading.Thread(target=_reader_thread,
  67. args=(filelike, out_queue)).start()
  68. def _read(self, rqueue):
  69. try:
  70. return rqueue.get(False)
  71. except queue.Empty:
  72. return None
  73. def _read_all(self, rqueue):
  74. reads = []
  75. while True:
  76. got = self._read(rqueue)
  77. if got is None:
  78. break
  79. reads.append(got)
  80. return reads
  81. def _read_stdout(self):
  82. return self._read_all(self._stdout)
  83. def _read_stderr(self):
  84. return self._read_all(self._stderr)
  85. def poll(self):
  86. exit_code = self._popen.poll()
  87. return self._popen.poll(), self._read_stdout(), self._read_stderr()
  88. def request_with_retry(func, *args, **kwargs):
  89. """Perform a requests http call, retrying with exponetial backoff.
  90. Args:
  91. func: An http-requesting function to call, like requests.post
  92. retries: Maximum retries before giving up.
  93. *args: passed through to func
  94. **kwargs: passed through to func
  95. """
  96. retries = kwargs.get('retries', 5)
  97. retry_delay = 2
  98. retry_count = 0
  99. while True:
  100. try:
  101. response = func(*args, **kwargs)
  102. response.raise_for_status()
  103. return True
  104. except (requests.exceptions.ConnectionError,
  105. requests.exceptions.HTTPError,
  106. requests.exceptions.Timeout) as e:
  107. logger.warning('requests_with_retry encountered retryable exception: %s. args: %s, kwargs: %s',
  108. e, args, kwargs)
  109. if retry_count == retries:
  110. return e
  111. retry_count += 1
  112. time.sleep(retry_delay)
  113. retry_delay *= 2
  114. except requests.exceptions.RequestException as e:
  115. logger.error(
  116. 'requests_with_retry encountered unretryable exception: %s', e)
  117. return e
Tip!

Press p or to see the previous file or, n or to see the next file

Comments

Loading...