DoOptimization.py
Go to the documentation of this file.
1 import Optimizations, subprocess, re, math
2 """
3 DoOptimization.py
4 Laura Fields
5 August 2014
6 
7 This is the main control script for running the optimization.
8 It does several tasks:
9 
10 - Itentifies new configurations that are ready to be generated
11 - Crates g4lbne macros for each new configuration and submits them
12 - Identifies configurations that have completed and merges output histos
13  (This step is done first so that the merged histos are available
14  for making future generations)
15 
16 """
17 
18 
19 import glob,sys, os
20 
21 if len(sys.argv) < 2:
22  print "Usage: "+sys.argv[0]+" <optname>"
23  sys.exit()
24 
25 optname = sys.argv[1]
26 
27 optimization = Optimizations.Optimization(optname)
28 
29 ##########################################
30 #
31 # Identify completed configurations and merge histograms
32 #
33 ##########################################
34 
35 configs = []
36 config_iter = optimization.getLastGoodConfig()+1
37 n_missing_configs = 0
38 while n_missing_configs < 15:
39  temp_histo_path = "/pnfs/dune/scratch/users/ljf26/fluxfiles/g4lbne/"+optimization.g4lbne_version+"/"+optimization.physics_list+"/Optimizations-"+optname+"-"+str(config_iter)
40 
41  if optimization.rhc_parameters_float_separate:
42  temp_histo_path = "/pnfs/dune/scratch/users/ljf26/fluxfiles/g4lbne/"+optimization.g4lbne_version+"/"+optimization.physics_list+"/Optimizations-"+optname+"FHC-"+str(config_iter)
43  if os.path.exists(temp_histo_path):
44  configs.append(temp_histo_path)
45  else:
46  n_missing_configs = n_missing_configs + 1
47 
48  config_iter = config_iter + 1
49 
50 print "Found",len(configs),"optimization histogram directories past the last good one"
51 
52 if len(configs)>100:
53  print "Found more than 100 configurations past the last good one."
54  print "This is a problem that needs manual intervention (like manually changing the recorded last good configuration)... quitting"
55  sys.exit()
56 
57 configs_to_merge = []
58 
59 for config in configs:
60 
61  # Skip over any configurations that were generated for a parameter scan
62  if "Scan" in config:
63  continue
64 
65  fhc_files = glob.glob(config+"/*/flux/histos/histos_*_neutrino_*_"+optimization.detector_location_name+"_fastmc.root")
66 
67  rhc_files = glob.glob(config.replace("FHC","RHC")+"/*/flux/histos/histos_*_antineutrino_*_"+optimization.detector_location_name+"_fastmc.root")
68 
69  configuration_number = config.split("/")[len(config.split("/"))-1].split("-")[2]
70 
71  print configuration_number,len(fhc_files),len(rhc_files)
72 
73  config = config.replace("/pnfs/dune/scratch",optimization.output_location)
74 
75  if len(fhc_files) >= 4 and len(rhc_files) >= 4:
76  merged_file = glob.glob(config+"/*/flux/histos_*_"+optimization.detector_location_name+"_fastmc.root")
77 
78  if optimization.rhc_parameters_float_separate:
79  merged_file = merged_file+glob.glob(config.replace("FHC","RHC")+"/*/flux/histos_*_"+optimization.detector_location_name+"_fastmc_merged.root")
80 
81  if not len(merged_file) == 2 :
82  configs_to_merge.append((configuration_number))
83 
84 print "Merging",len(configs_to_merge),"configurations"
85 for config_number in configs_to_merge:
86  if optimization.rhc_parameters_float_separate:
87  os.system("python ../../ProductionScripts/merge_histograms.py --mode neutrino --n 10000 -m Optimizations-"+optname+"FHC-"+str(config_number)+" -l "+optimization.detector_location_name+" --input_dir /pnfs/dune/scratch/users/ljf26/fluxfiles/g4lbne --output_dir "+optimization.output_location+"users/ljf26/fluxfiles/g4lbne -p "+optimization.physics_list)
88  os.system("python ../../ProductionScripts/merge_histograms.py --mode antineutrino --n 10000 -m Optimizations-"+optname+"RHC-"+str(config_number)+" -l "+optimization.detector_location_name+" --input_dir /pnfs/dune/scratch/users/ljf26/fluxfiles/g4lbne --output_dir "+optimization.output_location+"users/ljf26/fluxfiles/g4lbne -p "+optimization.physics_list)
89 
90  else:
91  os.system("python ../../ProductionScripts/merge_histograms.py --mode neutrino --n 10000 -m Optimizations-"+optname+"-"+str(config_number)+" -l "+optimization.detector_location_name+" --input_dir /pnfs/dune/scratch/users/ljf26/fluxfiles/g4lbne --output_dir "+optimization.output_location+"users/ljf26/fluxfiles/g4lbne -p "+optimization.physics_list)
92  os.system("python ../../ProductionScripts/merge_histograms.py --mode antineutrino --n 10000 -m Optimizations-"+optname+"-"+str(config_number)+" -l "+optimization.detector_location_name+" --input_dir /pnfs/dune/scratch/users/ljf26/fluxfiles/g4lbne --output_dir "+optimization.output_location+"users/ljf26/fluxfiles/g4lbne -p "+optimization.physics_list)
93 
94 ##########################################
95 #
96 # Identify completed configurations that can be used for next generation
97 #
98 ##########################################
99 # search for merged histos
100 configs_merged = []
101 n_dirs_with_merged_histos = 0
102 for config in configs:
103  if "Scan" in config:
104  continue
105 
106  config = config.replace("/pnfs/dune/scratch",optimization.output_location)
107 
108  merged_file = glob.glob(config+"/*/flux/histos_*_"+optimization.detector_location_name+"_fastmc.root")
109 
110  configuration_number = config.split("/")[len(config.split("/"))-1].split("-")[2]
111 
112  if len(merged_file) > 0 :
113  n_dirs_with_merged_histos = n_dirs_with_merged_histos + 1
114  configs_merged.append((configuration_number))
115 
116 print "Found ",len(configs_merged),"merged configurations"
117 configs_merged.sort(key=float)
118 
119 ##########################################
120 #
121 # Determine whether a new generation is ready to start
122 #
123 ##########################################
124 n_good_configs_in_range = 0
125 
126 first_config = optimization.getLastGoodConfig()+1
127 last_config = first_config+optimization.generation_size
128 
129 if len(configs_merged)>optimization.generation_size-10:
130 
131  first_config = first_config + optimization.generation_size
132  last_config = last_config + optimization.generation_size
133 else:
134  print "Not enough completed configurations to do another generation"
135 
136 ##########################################
137 #
138 # Make Macro files for next generation
139 #
140 ##########################################
141 print "first last ",first_config,last_config
142 
143 if(first_config == 1 and len(configs)==0):
144  print "Making first generation macro files...."
145 
146 if(len(configs_merged)>optimization.generation_size-10 or len(configs)==0):
147  optimization.populateMacros(first_config,last_config)
148 
149  # store a record of the last complete configuration
150  f = open(optimization.last_completed_config_file,"w")
151  f.write(str(first_config-1))
152  f.close()
153 
154 ##########################################
155 #
156 # Submit g4lbne jobs for next generation
157 #
158 ##########################################
159 resubmit_last_generation = False
160 
161 if(len(configs_merged) > (optimization.generation_size-10) or len(configs)==0 or resubmit_last_generation):
162  current_directory = os.getcwd()
163  os.chdir("../..")
164  for i in range(int(first_config),int(last_config)):
165 
166  # look up off axis angle for this configuration
167  detx = str(0)
168  dety = str(0)
169  detz = str(0)
170 
171  theta_y = optimization.getParameterValue("OffAxisAngleY",i)
172 
173  if(theta_y != -1):
174  detx = str(0)
175  dety = str(math.sin(theta_y)*129700000.0);
176  detz = str(math.cos(theta_y)*129700000.0);
177 
178  if optimization.rhc_parameters_float_separate:
179  os.system("python ProductionScripts/submit_flux.py -n 10000 -f 1 -l 5 --notuples --mode neutrino -m Optimizations-"+optname+"FHC-"+str(i)+" --detx "+detx+" --dety "+dety+" --detz "+detz+" -p "+optimization.physics_list)
180  os.system("python ProductionScripts/submit_flux.py -n 10000 -f 1 -l 5 --notuples --mode antineutrino -m Optimizations-"+optname+"RHC-"+str(i)+" --detx "+detx+" --dety "+dety+" --detz "+detz+" -p "+optimization.physics_list)
181 # os.system("python ProductionScripts/submit_flux.py -n 10000 -f 1 -l 5 --notuples --mode neutrino -m Optimizations-"+optname+"FHC-"+str(i)+" --output_dir /pnfs/dune/scratch/users/$USER/fluxfiles/g4lbne -p "+optimization.physics_list)
182 # os.system("python ProductionScripts/submit_flux.py -n 10000 -f 1 -l 5 --notuples --mode antineutrino -m Optimizations-"+optname+"RHC-"+str(i)+" --output_dir /pnfs/dune/scratch/users/$USER/fluxfiles/g4lbne -p "+optimization.physics_list)
183  else:
184  os.system("python ProductionScripts/submit_flux.py -n 10000 -f 1 -l 5 --notuples --mode neutrino -m Optimizations-"+optname+"-"+str(i)+" --detx "+detx+" --dety "+dety+" --detz "+detz+" -p "+optimization.physics_list)
185  os.system("python ProductionScripts/submit_flux.py -n 10000 -f 1 -l 5 --notuples --mode antineutrino -m Optimizations-"+optname+"-"+str(i)+" --detx "+detx+" --dety "+dety+" --detz "+detz+" -p "+optimization.physics_list)
186 # os.system("python ProductionScripts/submit_flux.py -n 10000 -f 1 -l 5 --notuples --mode antineutrino -m Optimizations-"+optname+"-"+str(i)+" --output_dir /pnfs/dune/scratch/users/$USER/fluxfiles/g4lbne -p "+optimization.physics_list)
187 # os.system("python ProductionScripts/submit_flux.py -n 10000 -f 1 -l 5 --notuples --mode antineutrino -m Optimizations-"+optname+"-"+str(i)+" --output_dir /pnfs/dune/scratch/users/$USER/fluxfiles/g4lbne -p "+optimization.physics_list)
188 
189  os.chdir("BeamSimStudies/GeneticOptimization")
190 
open
int open(const char *, int)
Opens a file descriptor.
keras_to_tensorflow.int
int
Definition: keras_to_tensorflow.py:69
cet::split
void split(std::string const &s, char c, OutIter dest)
Definition: split.h:35
if
if(!yymsg) yymsg
str
static QCString str
Definition: fortrancode.cpp:27098
Generated by   doxygen 1.8.11