Top

qnd module

Quick and Dirty TensorFlow command framework

"""Quick and Dirty TensorFlow command framework"""

from .flag import *
from .infer import def_infer
from .train_and_evaluate import def_train_and_evaluate
from .evaluate import def_evaluate
from .serve import def_serve

__all__ = ["FLAGS", "add_flag", "add_required_flag", "FlagAdder",
           "def_train_and_evaluate", "def_evaluate", "def_infer", "def_serve"]
__version__ = "0.1.3"

Module variables

var FLAGS

Functions

def add_flag(

name, *args, **kwargs)

Add a flag.

Added flags can be accessed by FLAGS module variable. (e.g. FLAGS.my_flag_name)

  • Args
    • name: Flag name. Real flag name will be "--{}".format(name).
    • *args, **kwargs: The rest arguments are the same as argparse.ArgumentParser.add_argument().
def add_flag(name, *args, **kwargs):
    """Add a flag.

    Added flags can be accessed by `FLAGS` module variable.
    (e.g. `FLAGS.my_flag_name`)

    - Args
        - `name`: Flag name. Real flag name will be `"--{}".format(name)`.
        - `*args`, `**kwargs`: The rest arguments are the same as
            `argparse.ArgumentParser.add_argument()`.
    """
    global _FLAG_NAMES

    if 'help' not in kwargs:
        kwargs['help'] = '(no description)'

    if name not in _FLAG_NAMES:
        _FLAG_NAMES.add(name)
        gargparse.add_argument("--" + name, *args, **kwargs)

def add_required_flag(

name, *args, **kwargs)

Add a required flag.

Its interface is the same as add_flag() but required=True is set by default.

def add_required_flag(name, *args, **kwargs):
    """Add a required flag.

    Its interface is the same as `add_flag()` but `required=True` is set by
    default.
    """
    add_flag(name, *args, required=True, **kwargs)

def def_evaluate(

batch_inputs=True, prepare_filename_queues=True)

Define evaluate() function.

See also help(def_evaluate()).

  • Args

    • batch_inputs: Same as def_train_and_evaluate()'s.
    • prepare_filename_queues: Same as def_train_and_evaluate()'s.
  • Returns

    • evaluate() function.
def def_evaluate(batch_inputs=True, prepare_filename_queues=True):
    """Define `evaluate()` function.

    See also `help(def_evaluate())`.

    - Args
        - `batch_inputs`: Same as `def_train_and_evaluate()`'s.
        - `prepare_filename_queues`: Same as `def_train_and_evaluate()`'s.

    - Returns
        - `evaluate()` function.
    """
    add_output_dir_flag()

    estimator = def_estimator(distributed=False)
    def_eval_input_fn = def_def_infer_input_fn(batch_inputs,
                                               prepare_filename_queues)

    def evaluate(model_fn, input_fn):
        """Evaluate a model with data sample fed by `input_fn`.

        - Args
            - `model_fn`: Same as `train_and_evaluate()`'s.
            - `input_fn`: Same as `eval_input_fn` argument of
                `train_and_evaluate()`.

        - Returns
            - Evaluation results. See `Evaluable` interface in TensorFlow.
        """
        return estimator(model_fn, FLAGS.output_dir).evaluate(
            input_fn=def_eval_input_fn(input_fn))

    return evaluate

def def_infer(

batch_inputs=True, prepare_filename_queues=True)

Define infer() function.

See also help(def_infer()).

  • Args

    • batch_inputs: Same as def_train_and_evaluate()'s.
    • prepare_filename_queues: Same as def_train_and_evaluate()'s.
  • Returns

    • infer() function.
def def_infer(batch_inputs=True, prepare_filename_queues=True):
    """Define `infer()` function.

    See also `help(def_infer())`.

    - Args
        - `batch_inputs`: Same as `def_train_and_evaluate()`'s.
        - `prepare_filename_queues`: Same as `def_train_and_evaluate()`'s.

    - Returns
        - `infer()` function.
    """
    add_output_dir_flag()

    estimator = def_estimator(distributed=False)
    def_infer_input_fn = def_def_infer_input_fn(batch_inputs,
                                                prepare_filename_queues)

    def infer(model_fn, input_fn):
        """Infer labels or regression values from features of samples fed by
        `input_fn`.

        - Args
            - `model_fn`: Same as `train_and_evaluate()`'s.
            - `input_fn`: Same as `train_input_fn` and `eval_input_fn`
                arguments of `train_and_evaluate()` but returns only features.

        - Returns
            - Generator of inferred label(s) or regression value(s) for each
                sample.
        """
        return estimator(model_fn, FLAGS.output_dir).predict(
            input_fn=def_infer_input_fn(input_fn))

    return infer

def def_serve(

)

Define serve() function.

See also help(def_serve()).

  • Returns
    • serve() function.
def def_serve():
    """Define `serve()` function.

    See also `help(def_serve())`.

    - Returns
        - `serve()` function.
    """
    add_output_dir_flag()

    create_estimator = def_estimator(distributed=False)

    def serve(model_fn, preprocess_fn, port=80):
        """Serve as a HTTP server.

        - Args
            - `model_fn`: Same as `train_and_evaluate()`'s.
            - `preprocess_fn`: A function to preprocess server request bodies
                in JSON.
        """
        estimator = create_estimator(model_fn, FLAGS.output_dir)

        class Handler(http.server.BaseHTTPRequestHandler):
            def do_POST(self):
                self.send_response(200)
                self.send_header('Content-type', 'application/json')
                self.end_headers()

                inputs = json.loads(self.rfile.read(
                    int(self.headers['Content-Length'])))

                predictions = _make_json_serializable(
                    estimator.predict(input_fn=lambda: preprocess_fn(inputs),
                                      as_iterable=False))

                logging.info('Prediction results: {}'.format(predictions))

                self.wfile.write(json.dumps(predictions).encode())

        http.server.HTTPServer(('', port), Handler).serve_forever()

    return serve

def def_train_and_evaluate(

batch_inputs=True, prepare_filename_queues=True, distributed=False)

Define train_and_evaluate() function.

See also help(def_train_and_evaluate()).

  • Args

    • batch_inputs: If True, create batches from Tensors returned from train_input_fn() and eval_input_fn() and feed them to a model.
    • prepare_filename_queues: If True, create filename queues for train and eval data based on file paths specified by command line arguments.
    • distributed: If True, configure command line arguments to train and evaluate models on a distributed system.
  • Returns

    • train_and_evaluate() function.
def def_train_and_evaluate(batch_inputs=True,
                           prepare_filename_queues=True,
                           distributed=False):
    """Define `train_and_evaluate()` function.

    See also `help(def_train_and_evaluate())`.

    - Args
        - `batch_inputs`: If `True`, create batches from Tensors returned from
            `train_input_fn()` and `eval_input_fn()` and feed them to a model.
        - `prepare_filename_queues`: If `True`, create filename queues for
            train and eval data based on file paths specified by command line
            arguments.
        - `distributed`: If `True`, configure command line arguments to train
            and evaluate models on a distributed system.

    - Returns
        - `train_and_evaluate()` function.
    """
    add_output_dir_flag()

    def_experiment_fn = def_def_experiment_fn(batch_inputs,
                                              prepare_filename_queues,
                                              distributed)

    def train_and_evaluate(model_fn,
                           train_input_fn,
                           eval_input_fn=None,
                           serving_input_fn=None):
        """Train and evaluate a model with features and targets fed by
        `input_fn`s.

        - Args
            - `model_fn`: A function to construct a model.
                - Types of its arguments must be one of the following:
                    - `Tensor, ...`
                    - `Tensor, ..., mode=ModeKeys`
                - Types of its return values must be one of the following:
                    - `Tensor, Tensor, Operation, eval_metric_ops=dict`
                        (predictions, loss, train_op, and eval_metric_ops (if any))
                    - `ModelFnOps`
            - `train_input_fn`, `eval_input_fn`: Functions to create input
                Tensors fed into the model. If `eval_input_fn` is `None`,
                `train_input_fn` will be used instead.
                - Types of its arguments must be one of the following:
                    - `QueueBase` (a filename queue)
                    - No argument if `prepare_filename_queues` of
                        `def_train_and_evaluate()` is `False`.
                - Types of its return values must be one of the following:
                    - `Tensor, Tensor` (features and targets)
                    - `dict, dict` (features and targets)
                        - The keys in `dict` objects must match with argument
                            names of `model_fn`.

        - Returns
            - Return value of `tf.contrib.learn.python.learn.learn_runner.run()`.
        """
        return run(def_experiment_fn(model_fn,
                                     train_input_fn,
                                     eval_input_fn,
                                     serving_input_fn),
                   FLAGS.output_dir)

    return train_and_evaluate

Classes

class FlagAdder

Manage addition of flags.

class FlagAdder:
    """Manage addition of flags."""

    def __init__(self):
        """Create a `FlagAdder` instance."""
        self._flags = []

    def add_flag(self, name, *args, **kwargs):
        """Add a flag.

        See `add_flag()`.
        """
        add_flag(name, *args, **kwargs)
        self._flags.append(kwargs.get("dest") or name)

    def add_required_flag(self, name, *args, **kwargs):
        """Add a required flag.

        See `add_required_flag()`.
        """
        self.add_flag(name, *args, required=True, **kwargs)

    @property
    def flags(self):
        """Get added flags.

        - Returns
            - `dict` of flag names to values added by a `FlagAdder` instance.
        """
        return {flag: getattr(FLAGS, flag) for flag in self._flags}

Ancestors (in MRO)

Static methods

def __init__(

self)

Create a FlagAdder instance.

def __init__(self):
    """Create a `FlagAdder` instance."""
    self._flags = []

def add_flag(

self, name, *args, **kwargs)

Add a flag.

See add_flag().

def add_flag(self, name, *args, **kwargs):
    """Add a flag.
    See `add_flag()`.
    """
    add_flag(name, *args, **kwargs)
    self._flags.append(kwargs.get("dest") or name)

def add_required_flag(

self, name, *args, **kwargs)

Add a required flag.

See add_required_flag().

def add_required_flag(self, name, *args, **kwargs):
    """Add a required flag.
    See `add_required_flag()`.
    """
    self.add_flag(name, *args, required=True, **kwargs)

Instance variables

var flags

Get added flags.

  • Returns
    • dict of flag names to values added by a FlagAdder instance.