-
Notifications
You must be signed in to change notification settings - Fork 0
/
run_experiments.py
109 lines (86 loc) · 2.93 KB
/
run_experiments.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
import os
from datetime import datetime
from pathlib import Path
from typing import Optional
from config import RESULTS_DIR, NoiseLevel, NUM_COMBINATIONS_PER_SUBSET, NoiseType
from experiments import (
ExperimentType,
run_experiment,
save_experiment_results,
)
def generic_run_experiment(
out_dir: Path,
experiments_types: list[ExperimentType],
noise_type: NoiseType,
should_show: bool = False,
should_save: bool = False,
experiment_name: Optional[str] = None,
):
experiment_name = experiment_name or noise_type.value
results_dir = out_dir / experiment_name
digits_to_test = list(range(10))
possible_num_examples_per_digit = [1, 5, 10, 30]
if noise_type == NoiseType.NONE:
possible_noise_levels = [NoiseLevel.NONE]
else:
possible_noise_levels = [
NoiseLevel.LOW,
NoiseLevel.MEDIUM,
]
for experiments_type in experiments_types:
results = run_experiment(
experiments_type,
digits_to_test,
noise_type,
possible_noise_levels,
possible_num_examples_per_digit,
NUM_COMBINATIONS_PER_SUBSET,
should_save,
should_show,
)
save_experiment_results(results_dir, results, experiments_type)
def run_golden_experiment(
out_dir: Path,
experiments_types: list[ExperimentType],
):
generic_run_experiment(
out_dir, experiments_types, NoiseType.NONE, experiment_name="golden"
)
def run_discrete_experiment(
out_dir: Path,
experiments_types: list[ExperimentType],
):
generic_run_experiment(out_dir, experiments_types, NoiseType.DISCRETE)
def run_continuous_experiment(
out_dir: Path,
experiments_types: list[ExperimentType],
):
generic_run_experiment(out_dir, experiments_types, NoiseType.CONTINUOUS)
def run_balanced_discrete_experiment(
out_dir: Path,
experiments_types: list[ExperimentType],
):
generic_run_experiment(out_dir, experiments_types, NoiseType.BALANCED_DISCRETE)
def run_balanced_continuous_experiment(
out_dir: Path,
experiments_types: list[ExperimentType],
):
generic_run_experiment(out_dir, experiments_types, NoiseType.BALANCED_CONTINUOUS)
def main():
# If GPU is available, use it
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
os.environ["XLA_PYTHON_CLIENT_MEM_FRACTION"] = "0.5" # Defaults to 0.9 * TOTAL_MEM
digits_dir = RESULTS_DIR / "digits" / datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
digits_dir.mkdir(exist_ok=True, parents=True)
experiments_types = [ExperimentType.MHN, ExperimentType.MDL_MHN]
experiments_funcs = [
run_golden_experiment,
run_discrete_experiment,
run_balanced_discrete_experiment,
run_continuous_experiment,
run_balanced_continuous_experiment,
]
for experiment_func in experiments_funcs:
experiment_func(digits_dir, experiments_types)
if __name__ == "__main__":
main()