qnd module
Quick and Dirty TensorFlow command framework
"""Quick and Dirty TensorFlow command framework"""
from .flag import *
from .infer import def_infer
from .train_and_evaluate import def_train_and_evaluate
from .evaluate import def_evaluate
from .serve import def_serve
__all__ = ["FLAGS", "add_flag", "add_required_flag", "FlagAdder",
"def_train_and_evaluate", "def_evaluate", "def_infer", "def_serve"]
__version__ = "0.1.3"
Module variables
var FLAGS
Functions
def add_flag(
name, *args, **kwargs)
Add a flag.
Added flags can be accessed by FLAGS
module variable.
(e.g. FLAGS.my_flag_name
)
- Args
name
: Flag name. Real flag name will be"--{}".format(name)
.*args
,**kwargs
: The rest arguments are the same asargparse.ArgumentParser.add_argument()
.
def add_flag(name, *args, **kwargs):
"""Add a flag.
Added flags can be accessed by `FLAGS` module variable.
(e.g. `FLAGS.my_flag_name`)
- Args
- `name`: Flag name. Real flag name will be `"--{}".format(name)`.
- `*args`, `**kwargs`: The rest arguments are the same as
`argparse.ArgumentParser.add_argument()`.
"""
global _FLAG_NAMES
if 'help' not in kwargs:
kwargs['help'] = '(no description)'
if name not in _FLAG_NAMES:
_FLAG_NAMES.add(name)
gargparse.add_argument("--" + name, *args, **kwargs)
def add_required_flag(
name, *args, **kwargs)
Add a required flag.
Its interface is the same as add_flag()
but required=True
is set by
default.
def add_required_flag(name, *args, **kwargs):
"""Add a required flag.
Its interface is the same as `add_flag()` but `required=True` is set by
default.
"""
add_flag(name, *args, required=True, **kwargs)
def def_evaluate(
batch_inputs=True, prepare_filename_queues=True)
Define evaluate()
function.
See also help(def_evaluate())
.
-
Args
batch_inputs
: Same asdef_train_and_evaluate()
's.prepare_filename_queues
: Same asdef_train_and_evaluate()
's.
-
Returns
evaluate()
function.
def def_evaluate(batch_inputs=True, prepare_filename_queues=True):
"""Define `evaluate()` function.
See also `help(def_evaluate())`.
- Args
- `batch_inputs`: Same as `def_train_and_evaluate()`'s.
- `prepare_filename_queues`: Same as `def_train_and_evaluate()`'s.
- Returns
- `evaluate()` function.
"""
add_output_dir_flag()
estimator = def_estimator(distributed=False)
def_eval_input_fn = def_def_infer_input_fn(batch_inputs,
prepare_filename_queues)
def evaluate(model_fn, input_fn):
"""Evaluate a model with data sample fed by `input_fn`.
- Args
- `model_fn`: Same as `train_and_evaluate()`'s.
- `input_fn`: Same as `eval_input_fn` argument of
`train_and_evaluate()`.
- Returns
- Evaluation results. See `Evaluable` interface in TensorFlow.
"""
return estimator(model_fn, FLAGS.output_dir).evaluate(
input_fn=def_eval_input_fn(input_fn))
return evaluate
def def_infer(
batch_inputs=True, prepare_filename_queues=True)
Define infer()
function.
See also help(def_infer())
.
-
Args
batch_inputs
: Same asdef_train_and_evaluate()
's.prepare_filename_queues
: Same asdef_train_and_evaluate()
's.
-
Returns
infer()
function.
def def_infer(batch_inputs=True, prepare_filename_queues=True):
"""Define `infer()` function.
See also `help(def_infer())`.
- Args
- `batch_inputs`: Same as `def_train_and_evaluate()`'s.
- `prepare_filename_queues`: Same as `def_train_and_evaluate()`'s.
- Returns
- `infer()` function.
"""
add_output_dir_flag()
estimator = def_estimator(distributed=False)
def_infer_input_fn = def_def_infer_input_fn(batch_inputs,
prepare_filename_queues)
def infer(model_fn, input_fn):
"""Infer labels or regression values from features of samples fed by
`input_fn`.
- Args
- `model_fn`: Same as `train_and_evaluate()`'s.
- `input_fn`: Same as `train_input_fn` and `eval_input_fn`
arguments of `train_and_evaluate()` but returns only features.
- Returns
- Generator of inferred label(s) or regression value(s) for each
sample.
"""
return estimator(model_fn, FLAGS.output_dir).predict(
input_fn=def_infer_input_fn(input_fn))
return infer
def def_serve(
)
Define serve()
function.
See also help(def_serve())
.
- Returns
serve()
function.
def def_serve():
"""Define `serve()` function.
See also `help(def_serve())`.
- Returns
- `serve()` function.
"""
add_output_dir_flag()
create_estimator = def_estimator(distributed=False)
def serve(model_fn, preprocess_fn, port=80):
"""Serve as a HTTP server.
- Args
- `model_fn`: Same as `train_and_evaluate()`'s.
- `preprocess_fn`: A function to preprocess server request bodies
in JSON.
"""
estimator = create_estimator(model_fn, FLAGS.output_dir)
class Handler(http.server.BaseHTTPRequestHandler):
def do_POST(self):
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
inputs = json.loads(self.rfile.read(
int(self.headers['Content-Length'])))
predictions = _make_json_serializable(
estimator.predict(input_fn=lambda: preprocess_fn(inputs),
as_iterable=False))
logging.info('Prediction results: {}'.format(predictions))
self.wfile.write(json.dumps(predictions).encode())
http.server.HTTPServer(('', port), Handler).serve_forever()
return serve
def def_train_and_evaluate(
batch_inputs=True, prepare_filename_queues=True, distributed=False)
Define train_and_evaluate()
function.
See also help(def_train_and_evaluate())
.
-
Args
batch_inputs
: IfTrue
, create batches from Tensors returned fromtrain_input_fn()
andeval_input_fn()
and feed them to a model.prepare_filename_queues
: IfTrue
, create filename queues for train and eval data based on file paths specified by command line arguments.distributed
: IfTrue
, configure command line arguments to train and evaluate models on a distributed system.
-
Returns
train_and_evaluate()
function.
def def_train_and_evaluate(batch_inputs=True,
prepare_filename_queues=True,
distributed=False):
"""Define `train_and_evaluate()` function.
See also `help(def_train_and_evaluate())`.
- Args
- `batch_inputs`: If `True`, create batches from Tensors returned from
`train_input_fn()` and `eval_input_fn()` and feed them to a model.
- `prepare_filename_queues`: If `True`, create filename queues for
train and eval data based on file paths specified by command line
arguments.
- `distributed`: If `True`, configure command line arguments to train
and evaluate models on a distributed system.
- Returns
- `train_and_evaluate()` function.
"""
add_output_dir_flag()
def_experiment_fn = def_def_experiment_fn(batch_inputs,
prepare_filename_queues,
distributed)
def train_and_evaluate(model_fn,
train_input_fn,
eval_input_fn=None,
serving_input_fn=None):
"""Train and evaluate a model with features and targets fed by
`input_fn`s.
- Args
- `model_fn`: A function to construct a model.
- Types of its arguments must be one of the following:
- `Tensor, ...`
- `Tensor, ..., mode=ModeKeys`
- Types of its return values must be one of the following:
- `Tensor, Tensor, Operation, eval_metric_ops=dict`
(predictions, loss, train_op, and eval_metric_ops (if any))
- `ModelFnOps`
- `train_input_fn`, `eval_input_fn`: Functions to create input
Tensors fed into the model. If `eval_input_fn` is `None`,
`train_input_fn` will be used instead.
- Types of its arguments must be one of the following:
- `QueueBase` (a filename queue)
- No argument if `prepare_filename_queues` of
`def_train_and_evaluate()` is `False`.
- Types of its return values must be one of the following:
- `Tensor, Tensor` (features and targets)
- `dict, dict` (features and targets)
- The keys in `dict` objects must match with argument
names of `model_fn`.
- Returns
- Return value of `tf.contrib.learn.python.learn.learn_runner.run()`.
"""
return run(def_experiment_fn(model_fn,
train_input_fn,
eval_input_fn,
serving_input_fn),
FLAGS.output_dir)
return train_and_evaluate
Classes
class FlagAdder
Manage addition of flags.
class FlagAdder:
"""Manage addition of flags."""
def __init__(self):
"""Create a `FlagAdder` instance."""
self._flags = []
def add_flag(self, name, *args, **kwargs):
"""Add a flag.
See `add_flag()`.
"""
add_flag(name, *args, **kwargs)
self._flags.append(kwargs.get("dest") or name)
def add_required_flag(self, name, *args, **kwargs):
"""Add a required flag.
See `add_required_flag()`.
"""
self.add_flag(name, *args, required=True, **kwargs)
@property
def flags(self):
"""Get added flags.
- Returns
- `dict` of flag names to values added by a `FlagAdder` instance.
"""
return {flag: getattr(FLAGS, flag) for flag in self._flags}
Ancestors (in MRO)
- FlagAdder
- builtins.object
Static methods
def __init__(
self)
Create a FlagAdder
instance.
def __init__(self):
"""Create a `FlagAdder` instance."""
self._flags = []
def add_flag(
self, name, *args, **kwargs)
Add a flag.
See add_flag()
.
def add_flag(self, name, *args, **kwargs):
"""Add a flag.
See `add_flag()`.
"""
add_flag(name, *args, **kwargs)
self._flags.append(kwargs.get("dest") or name)
def add_required_flag(
self, name, *args, **kwargs)
Add a required flag.
See add_required_flag()
.
def add_required_flag(self, name, *args, **kwargs):
"""Add a required flag.
See `add_required_flag()`.
"""
self.add_flag(name, *args, required=True, **kwargs)
Instance variables
var flags
Get added flags.
- Returns
dict
of flag names to values added by aFlagAdder
instance.