Skip to content

change the write lp and mps extension so it takes a directory name #537

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions doc/src/extensions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -329,17 +329,18 @@ If some variables have zero probability in all scenarios, then you will need to
all variable probabilities! So you might want to set this to False to verify that the probabilities sum to one
only for the Vars you expect before setting it to True.

Scenario_lp_mps_writer
----------------------
Scenario_lp_mps_writer_dir
--------------------------

This extension writes an lp file and an mps file with the model as well as a
json file with (a) list(s) of scenario tree node names and
nonanticaptive variables for each scenario before the iteration zero
solve of PH or APH. Note that for two-stage problems, all json files
will be the same. See ``mpisppy.generic_cylinders.py`` for an example
of use. In that program it is activated with the
``--scenario-lp-mps-writer`` option. Note that it
writes the files to the current working directory and for each scenario
``--scenario-lp-mps-writerdir`` option that specifies a directory that
does not exist. The extension
writes the files to this directory and for each scenario
the base name of the three files written is the scenario name.

Unless you know exactly why you need this, you probably don't.
8 changes: 4 additions & 4 deletions examples/sizes/mps_demo.bash
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
# Note: if you actually have a Pyomo model, you probably don't want to do
# it this way since you would have had to have written most of the
# functions (e.g. scenario_creator) anyway.
# If you are using some other AML, then you migth want to use the second
# If you are using some other AML, then you might want to use the second
# command line to read the files you wrote with your AML and
# you can use the first command to write files as an example of the format
# for the json files.
Expand All @@ -15,9 +15,9 @@ set -e

SOLVER=cplex

# assumes we are in the sizes directory and don't mind polluting it with 6 files
python ../../mpisppy/generic_cylinders.py --module-name sizes_expression --num-scens 3 --default-rho 1 --solver-name ${SOLVER} --max-iterations 0 --scenario-lp-mps-files
# assumes we are in the sizes directory and don't mind polluting it with a directory
python ../../mpisppy/generic_cylinders.py --module-name sizes_expression --num-scens 3 --default-rho 1 --solver-name ${SOLVER} --max-iterations 0 --write-scenario-lp-mps-files-dir _delme_lp_mps_dir

# By specifying the module to be mps_module we will read files for the problem
# from the specified mps-files-directory.
mpiexec -np 3 python -m mpi4py ../../mpisppy/generic_cylinders.py --module-name ../../mpisppy/utils/mps_module --xhatshuffle --lagrangian --default-rho 1 --solver-name ${SOLVER} --max-iterations 10 --mps-files-directory=.
mpiexec -np 3 python -m mpi4py ../../mpisppy/generic_cylinders.py --module-name ../../mpisppy/utils/mps_module --xhatshuffle --lagrangian --default-rho 1 --solver-name ${SOLVER} --max-iterations 10 --mps-files-directory=_delme_lp_mps_dir
25 changes: 21 additions & 4 deletions mpisppy/extensions/scenario_lp_mps_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
the nonant structure for each scenario (yes, for two stage problems this
json file will be the same for all scenarios.)
"""

import os
import json
import mpisppy.extensions.extension
import pyomo.core.base.label as pyomo_label
Expand All @@ -25,20 +25,37 @@ class Scenario_lp_mps_files(mpisppy.extensions.extension.Extension):

def __init__(self, ph):
self.ph = ph
self.dirname = self.ph.options["write_lp_mps_extension_options"].get("write_scenario_lp_mps_files_dir")
if self.dirname is None:
raise RuntimeError("Scenario_lp_mps_files extension"
" cannot be used without the"
" write_scenario_lp_mps_files_dir option")
os.makedirs(self.dirname, exist_ok=False)


def pre_iter0(self):
dn = self.dirname # typing aid
for k, s in self.ph.local_subproblems.items():
s.write(f"{k}.lp", io_options={'symbolic_solver_labels': True})
s.write(f"{k}.mps", io_options={'symbolic_solver_labels': True})
s.write(os.path.join(dn, f"{k}.lp"), io_options={'symbolic_solver_labels': True})
s.write(os.path.join(dn, f"{k}.mps"), io_options={'symbolic_solver_labels': True})
scenData = {"name": s.name, "scenProb": s._mpisppy_probability}
scenDict = {"scenarioData": scenData}
treeData = dict()
rhoList = list()
for nd in s._mpisppy_node_list:
treeData[nd.name] = {"condProb": nd.cond_prob}
treeData[nd.name].update({"nonAnts": [lpize(var.name) for var in nd.nonant_vardata_list]})
rhoList.extend((lpize(var.name), s._mpisppy_model.rho[(nd.name, i)]._value)\
for i,var in enumerate(nd.nonant_vardata_list))

scenDict["treeData"] = treeData
with open(f"{k}_nonants.json", "w") as jfile:
with open(os.path.join(dn, f"{k}_nonants.json"), "w") as jfile:
json.dump(scenDict, jfile, indent=2)
# Note that for two stage problems, all rho files will be the same
with open(os.path.join(dn, f"{k}_rho.csv"), "w") as rfile:
rfile.write("varname,rho\n")
for name, rho in rhoList:
rfile.write(f"{name},{rho}\n")

def post_iter0(self):
return
Expand Down
10 changes: 6 additions & 4 deletions mpisppy/generic_cylinders.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,10 @@ def _parse_args(m):
description="The string used for a directory of ouput along with a csv and an npv file (default None, which means no soltion output)",
domain=str,
default=None)
cfg.add_to_config(name="write_scenario_lp_mps_files",
cfg.add_to_config(name="write_scenario_lp_mps_files_dir",
description="Invokes an extension that writes an model lp file, mps file and a nonants json file for each scenario before iteration 0",
domain=bool,
default=False)
domain=str,
default=None)

m.inparser_adder(cfg)
# many models, e.g., farmer, need num_scens_required
Expand Down Expand Up @@ -239,8 +239,10 @@ def _do_decomp(module, cfg, scenario_creator, scenario_creator_kwargs, scenario_
ext_classes.append(Gradient_extension)
hub_dict['opt_kwargs']['options']['gradient_extension_options'] = {'cfg': cfg}

if cfg.write_scenario_lp_mps_files:
if cfg.write_scenario_lp_mps_files_dir is not None:
ext_classes.append(Scenario_lp_mps_files)
hub_dict['opt_kwargs']['options']["write_lp_mps_extension_options"]\
= {"write_scenario_lp_mps_files_dir": cfg.write_scenario_lp_mps_files_dir}

if cfg.W_and_xbar_reader:
ext_classes.append(WXBarReader)
Expand Down
19 changes: 9 additions & 10 deletions mpisppy/tests/test_ef_ph.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import os
import glob
import json
#import math
import shutil
import unittest
import pandas as pd
import pyomo.environ as pyo
Expand Down Expand Up @@ -402,13 +402,16 @@ def test_xhat_extension(self):
@unittest.skipIf(not pyo.SolverFactory('glpk').available(),
"glpk is not available")
def test_scenario_lpwriter_extension(self):
print("test scenarip_lpwriter")
from mpisppy.extensions.scenario_lp_mps_files import Scenario_lp_mps_files
tdir = "_delme_test_write_mp_mps_dir"
if os.path.exists(tdir):
shutil.rmtree(tdir)
options = self._copy_of_base_options()
options["iter0_solver_options"] = {"mipgap": 0.1}
options["PHIterLimit"] = 0
options["solver_name"] = "glpk"
options["tee_rank0_solves"] = True
options["write_lp_mps_extension_options"]\
= {"write_scenario_lp_mps_files_dir": tdir}

ph = mpisppy.opt.ph.PH(
options,
Expand All @@ -420,22 +423,18 @@ def test_scenario_lpwriter_extension(self):
)
conv, basic_obj, tbound = ph.ph_main()
# The idea is to detect a change in Pyomo's writing of lp files
with open("Scenario1_nonants.json", "r") as jfile:
with open(os.path.join(tdir, "Scenario1_nonants.json"), "r") as jfile:
nonants_by_node = json.load(jfile)
# vname is first name in the file
vname = nonants_by_node["treeData"]["ROOT"]["nonAnts"][0]
gotit = False
with open("Scenario1.lp", 'r') as lpfile:
with open(os.path.join(tdir, "Scenario1.lp"), 'r') as lpfile:
for line in lpfile:
if vname in line:
gotit = True
break
assert gotit, f"The first nonant in Scenario1_nonants.json ({vname}) not found in Scenario1.lp"
print(" deleting Scenario*.p and Scenario*_nonants.json")
for fn in glob.glob("Scenario*.lp"):
os.remove(fn)
for fn in glob.glob("Scenario*_nonants.json"):
os.remove(fn)
print(f" deleting f{tdir}")


@unittest.skipIf(not solver_available,
Expand Down
2 changes: 1 addition & 1 deletion mpisppy/tests/test_mps.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

solver_available, solver_name, persistent_available, persistent_solver_name= get_solver(persistent_OK=False)

class TestStochAdmmWrapper(unittest.TestCase):
class TestMPSReader(unittest.TestCase):
def setUp(self):
pass

Expand Down
2 changes: 1 addition & 1 deletion mpisppy/utils/mps_module.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,11 @@ def scenario_creator(sname, cfg=None):
scenProb = nonantDict["scenarioData"]["scenProb"]
except Exception as e:
raise RuntimeError(f'Error getting scenProb from {jsonPath}: {e}')
assert "ROOT" in nonantDict, f'"ROOT" must be top node in {jsonPath}'
treeNodes = list()
parent_ndn = None # counting on the json file to have ordered nodes
stage = 1
treeDict = nonantDict["treeData"]
assert "ROOT" in treeDict, f'"ROOT" must be top node in {jsonPath}'
for ndn in treeDict:
cp = treeDict[ndn]["condProb"]
nonants = [model.\
Expand Down
Loading