|
@@ -2,13 +2,13 @@ import math
|
|
import time
|
|
import time
|
|
from functools import lru_cache
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from pathlib import Path
|
|
-from typing import Mapping, Optional, Tuple, Union, List, Dict
|
|
|
|
|
|
+from typing import Mapping, Optional, Tuple, Union, List, Dict, Any
|
|
from zipfile import ZipFile
|
|
from zipfile import ZipFile
|
|
import os
|
|
import os
|
|
from jsonschema import validate
|
|
from jsonschema import validate
|
|
import tarfile
|
|
import tarfile
|
|
from PIL import Image, ExifTags
|
|
from PIL import Image, ExifTags
|
|
-
|
|
|
|
|
|
+import re
|
|
import torch
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.nn as nn
|
|
|
|
|
|
@@ -168,18 +168,77 @@ def tensor_container_to_device(obj: Union[torch.Tensor, tuple, list, dict], devi
|
|
return obj
|
|
return obj
|
|
|
|
|
|
|
|
|
|
|
|
+def fuzzy_keys(params: Mapping) -> List[str]:
|
|
|
|
+ """
|
|
|
|
+ Returns params.key() removing leading and trailing white space, lower-casing and dropping symbols.
|
|
|
|
+ :param params: Mapping, the mapping containing the keys to be returned.
|
|
|
|
+ :return: List[str], list of keys as discussed above.
|
|
|
|
+ """
|
|
|
|
+ return [fuzzy_str(s) for s in params.keys()]
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def fuzzy_str(s: str):
|
|
|
|
+ """
|
|
|
|
+ Returns s removing leading and trailing white space, lower-casing and drops
|
|
|
|
+ :param s: str, string to apply the manipulation discussed above.
|
|
|
|
+ :return: str, s after the manipulation discussed above.
|
|
|
|
+ """
|
|
|
|
+ return re.sub(r"[^\w]", "", s).replace("_", "").lower()
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def _get_fuzzy_attr_map(params):
|
|
|
|
+ return {fuzzy_str(a): a for a in params.__dir__()}
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def _has_fuzzy_attr(params, name):
|
|
|
|
+ return fuzzy_str(name) in _get_fuzzy_attr_map(params)
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def get_fuzzy_mapping_param(name: str, params: Mapping):
|
|
|
|
+ """
|
|
|
|
+ Returns parameter value, with key=name with no sensitivity to lowercase, uppercase and symbols.
|
|
|
|
+ :param name: str, the key in params which is fuzzy-matched and retruned.
|
|
|
|
+ :param params: Mapping, the mapping containing param.
|
|
|
|
+ :return:
|
|
|
|
+ """
|
|
|
|
+ fuzzy_params = {fuzzy_str(key): params[key] for key in params.keys()}
|
|
|
|
+ return fuzzy_params[fuzzy_str(name)]
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def get_fuzzy_attr(params: Any, name: str):
|
|
|
|
+ """
|
|
|
|
+ Returns attribute (same functionality as getattr), but non sensitive to symbols, uppercase and lowercase.
|
|
|
|
+ :param params: Any, any object which wed looking for the attribute name in.
|
|
|
|
+ :param name: str, the attribute of param to be returned.
|
|
|
|
+ :return: Any, the attribute value or None when not fuzzy matching of the attribute is found
|
|
|
|
+ """
|
|
|
|
+ return getattr(params, _get_fuzzy_attr_map(params)[fuzzy_str(name)])
|
|
|
|
+
|
|
|
|
+
|
|
|
|
+def fuzzy_idx_in_list(name: str, lst: List[str]) -> int:
|
|
|
|
+ """
|
|
|
|
+ Returns the index of name in lst, with non sensitivity to symbols, uppercase and lowercase.
|
|
|
|
+ :param name: str, the name to be searched in lst.
|
|
|
|
+ :param lst: List[str], the list as described above.
|
|
|
|
+ :return: int, index of name in lst in the matter discussed above.
|
|
|
|
+ """
|
|
|
|
+ return [fuzzy_str(x) for x in lst].index(fuzzy_str(name))
|
|
|
|
+
|
|
|
|
+
|
|
def get_param(params, name, default_val=None):
|
|
def get_param(params, name, default_val=None):
|
|
"""
|
|
"""
|
|
- Retrieves a param from a parameter object/dict. If the parameter does not exist, will return default_val.
|
|
|
|
|
|
+ Retrieves a param from a parameter object/dict . If the parameter does not exist, will return default_val.
|
|
In case the default_val is of type dictionary, and a value is found in the params - the function
|
|
In case the default_val is of type dictionary, and a value is found in the params - the function
|
|
will return the default value dictionary with internal values overridden by the found value
|
|
will return the default value dictionary with internal values overridden by the found value
|
|
|
|
+ IMPORTANT: Not sensitive to lowercase, uppercase and symbols.
|
|
|
|
|
|
i.e.
|
|
i.e.
|
|
default_opt_params = {'lr':0.1, 'momentum':0.99, 'alpha':0.001}
|
|
default_opt_params = {'lr':0.1, 'momentum':0.99, 'alpha':0.001}
|
|
training_params = {'optimizer_params': {'lr':0.0001}, 'batch': 32 .... }
|
|
training_params = {'optimizer_params': {'lr':0.0001}, 'batch': 32 .... }
|
|
- get_param(training_params, name='optimizer_params', default_val=default_opt_params)
|
|
|
|
|
|
+ get_param(training_params, name='OptimizerParams', default_val=default_opt_params)
|
|
will return {'lr':0.0001, 'momentum':0.99, 'alpha':0.001}
|
|
will return {'lr':0.0001, 'momentum':0.99, 'alpha':0.001}
|
|
|
|
|
|
|
|
+
|
|
:param params: an object (typically HpmStruct) or a dict holding the params
|
|
:param params: an object (typically HpmStruct) or a dict holding the params
|
|
:param name: name of the searched parameter
|
|
:param name: name of the searched parameter
|
|
:param default_val: assumed to be the same type as the value searched in the params
|
|
:param default_val: assumed to be the same type as the value searched in the params
|
|
@@ -187,19 +246,24 @@ def get_param(params, name, default_val=None):
|
|
"""
|
|
"""
|
|
if isinstance(params, Mapping):
|
|
if isinstance(params, Mapping):
|
|
if name in params:
|
|
if name in params:
|
|
- if isinstance(default_val, Mapping):
|
|
|
|
- return {**default_val, **params[name]}
|
|
|
|
- else:
|
|
|
|
- return params[name]
|
|
|
|
|
|
+ param_val = params[name]
|
|
|
|
+
|
|
|
|
+ elif fuzzy_str(name) in fuzzy_keys(params):
|
|
|
|
+ param_val = get_fuzzy_mapping_param(name, params)
|
|
|
|
+
|
|
else:
|
|
else:
|
|
- return default_val
|
|
|
|
|
|
+ param_val = default_val
|
|
elif hasattr(params, name):
|
|
elif hasattr(params, name):
|
|
- if isinstance(default_val, Mapping):
|
|
|
|
- return {**default_val, **getattr(params, name)}
|
|
|
|
- else:
|
|
|
|
- return getattr(params, name)
|
|
|
|
|
|
+ param_val = getattr(params, name)
|
|
|
|
+ elif _has_fuzzy_attr(params, name):
|
|
|
|
+ param_val = get_fuzzy_attr(params, name)
|
|
|
|
+ else:
|
|
|
|
+ param_val = default_val
|
|
|
|
+
|
|
|
|
+ if isinstance(default_val, Mapping):
|
|
|
|
+ return {**default_val, **param_val}
|
|
else:
|
|
else:
|
|
- return default_val
|
|
|
|
|
|
+ return param_val
|
|
|
|
|
|
|
|
|
|
def static_vars(**kwargs):
|
|
def static_vars(**kwargs):
|