From 284ee194b115b98b0060998f4dcdf48d2049aab6 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Mon, 19 Jun 2017 14:38:15 +0100
Subject: [PATCH 001/115] JSON update

---
 lib/json/json.hpp | 3457 +++++++++++++++++++++++++++------------------
 1 file changed, 2092 insertions(+), 1365 deletions(-)

diff --git a/lib/json/json.hpp b/lib/json/json.hpp
index d5dc111e..6dfc1831 100644
--- a/lib/json/json.hpp
+++ b/lib/json/json.hpp
@@ -1,7 +1,7 @@
 /*
     __ _____ _____ _____
  __|  |   __|     |   | |  JSON for Modern C++
-|  |  |__   |  |  | | | |  version 2.0.10
+|  |  |__   |  |  | | | |  version 2.1.1
 |_____|_____|_____|_|___|  https://github.com/nlohmann/json
 
 Licensed under the MIT License <http://opensource.org/licenses/MIT>.
@@ -28,22 +28,23 @@ SOFTWARE.
 
 #ifndef NLOHMANN_JSON_HPP
 #define NLOHMANN_JSON_HPP
-#include <Grid/DisableWarnings.h>
-#include <algorithm> // all_of, for_each, transform
+
+#include <algorithm> // all_of, copy, fill, find, for_each, none_of, remove, reverse, transform
 #include <array> // array
 #include <cassert> // assert
 #include <cctype> // isdigit
 #include <ciso646> // and, not, or
-#include <cmath> // isfinite, ldexp, signbit
+#include <cmath> // isfinite, labs, ldexp, signbit
 #include <cstddef> // nullptr_t, ptrdiff_t, size_t
 #include <cstdint> // int64_t, uint64_t
-#include <cstdlib> // strtod, strtof, strtold, strtoul
+#include <cstdlib> // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull
 #include <cstring> // strlen
+#include <forward_list> // forward_list
 #include <functional> // function, hash, less
 #include <initializer_list> // initializer_list
 #include <iomanip> // setw
 #include <iostream> // istream, ostream
-#include <iterator> // advance, begin, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
+#include <iterator> // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
 #include <limits> // numeric_limits
 #include <locale> // locale
 #include <map> // map
@@ -52,19 +53,17 @@ SOFTWARE.
 #include <sstream> // stringstream
 #include <stdexcept> // domain_error, invalid_argument, out_of_range
 #include <string> // getline, stoi, string, to_string
-#include <type_traits> // add_pointer, enable_if, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_floating_point, is_integral, is_nothrow_move_assignable, std::is_nothrow_move_constructible, std::is_pointer, std::is_reference, std::is_same, remove_const, remove_pointer, remove_reference
+#include <type_traits> // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type
 #include <utility> // declval, forward, make_pair, move, pair, swap
 #include <vector> // vector
 
 // exclude unsupported compilers
 #if defined(__clang__)
-    #define CLANG_VERSION (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__)
-    #if CLANG_VERSION < 30400
+    #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400
         #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
     #endif
 #elif defined(__GNUC__)
-    #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
-    #if GCC_VERSION < 40800
+    #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900
         #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
     #endif
 #endif
@@ -90,6 +89,17 @@ SOFTWARE.
     #define JSON_DEPRECATED
 #endif
 
+// allow to disable exceptions
+#if not defined(JSON_NOEXCEPTION) || defined(__EXCEPTIONS)
+    #define JSON_THROW(exception) throw exception
+    #define JSON_TRY try
+    #define JSON_CATCH(exception) catch(exception)
+#else
+    #define JSON_THROW(exception) std::abort()
+    #define JSON_TRY if(true)
+    #define JSON_CATCH(exception) if(false)
+#endif
+
 /*!
 @brief namespace for Niels Lohmann
 @see https://github.com/nlohmann
@@ -98,38 +108,842 @@ SOFTWARE.
 namespace nlohmann
 {
 
-
 /*!
 @brief unnamed namespace with internal helper functions
+
+This namespace collects some functions that could not be defined inside the
+@ref basic_json class.
+
+@since version 2.1.0
+*/
+namespace detail
+{
+///////////////////////////
+// JSON type enumeration //
+///////////////////////////
+
+/*!
+@brief the JSON type enumeration
+
+This enumeration collects the different JSON types. It is internally used to
+distinguish the stored values, and the functions @ref basic_json::is_null(),
+@ref basic_json::is_object(), @ref basic_json::is_array(),
+@ref basic_json::is_string(), @ref basic_json::is_boolean(),
+@ref basic_json::is_number() (with @ref basic_json::is_number_integer(),
+@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()),
+@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and
+@ref basic_json::is_structured() rely on it.
+
+@note There are three enumeration entries (number_integer, number_unsigned, and
+number_float), because the library distinguishes these three types for numbers:
+@ref basic_json::number_unsigned_t is used for unsigned integers,
+@ref basic_json::number_integer_t is used for signed integers, and
+@ref basic_json::number_float_t is used for floating-point numbers or to
+approximate integers which do not fit in the limits of their respective type.
+
+@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON
+value with the default value for a given type
+
 @since version 1.0.0
 */
-namespace
+enum class value_t : uint8_t
 {
+    null,            ///< null value
+    object,          ///< object (unordered set of name/value pairs)
+    array,           ///< array (ordered collection of values)
+    string,          ///< string value
+    boolean,         ///< boolean value
+    number_integer,  ///< number value (signed integer)
+    number_unsigned, ///< number value (unsigned integer)
+    number_float,    ///< number value (floating-point)
+    discarded        ///< discarded by the the parser callback function
+};
+
+/*!
+@brief comparison operator for JSON types
+
+Returns an ordering that is similar to Python:
+- order: null < boolean < number < object < array < string
+- furthermore, each type is not smaller than itself
+
+@since version 1.0.0
+*/
+inline bool operator<(const value_t lhs, const value_t rhs) noexcept
+{
+    static constexpr std::array<uint8_t, 8> order = {{
+            0, // null
+            3, // object
+            4, // array
+            5, // string
+            1, // boolean
+            2, // integer
+            2, // unsigned
+            2, // float
+        }
+    };
+
+    // discarded values are not comparable
+    if (lhs == value_t::discarded or rhs == value_t::discarded)
+    {
+        return false;
+    }
+
+    return order[static_cast<std::size_t>(lhs)] <
+           order[static_cast<std::size_t>(rhs)];
+}
+
+
+/////////////
+// helpers //
+/////////////
+
+// alias templates to reduce boilerplate
+template<bool B, typename T = void>
+using enable_if_t = typename std::enable_if<B, T>::type;
+
+template<typename T>
+using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
+
+// taken from http://stackoverflow.com/a/26936864/266378
+template<typename T>
+using is_unscoped_enum =
+    std::integral_constant<bool, std::is_convertible<T, int>::value and
+    std::is_enum<T>::value>;
+
+/*
+Implementation of two C++17 constructs: conjunction, negation. This is needed
+to avoid evaluating all the traits in a condition
+
+For example: not std::is_same<void, T>::value and has_value_type<T>::value
+will not compile when T = void (on MSVC at least). Whereas
+conjunction<negation<std::is_same<void, T>>, has_value_type<T>>::value will
+stop evaluating if negation<...>::value == false
+
+Please note that those constructs must be used with caution, since symbols can
+become very long quickly (which can slow down compilation and cause MSVC
+internal compiler errors). Only use it when you have to (see example ahead).
+*/
+template<class...> struct conjunction : std::true_type {};
+template<class B1> struct conjunction<B1> : B1 {};
+template<class B1, class... Bn>
+struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {};
+
+template<class B> struct negation : std::integral_constant < bool, !B::value > {};
+
+// dispatch utility (taken from ranges-v3)
+template<unsigned N> struct priority_tag : priority_tag < N - 1 > {};
+template<> struct priority_tag<0> {};
+
+
+//////////////////
+// constructors //
+//////////////////
+
+template<value_t> struct external_constructor;
+
+template<>
+struct external_constructor<value_t::boolean>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept
+    {
+        j.m_type = value_t::boolean;
+        j.m_value = b;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::string>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s)
+    {
+        j.m_type = value_t::string;
+        j.m_value = s;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::number_float>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept
+    {
+        // replace infinity and NAN by null
+        if (not std::isfinite(val))
+        {
+            j = BasicJsonType{};
+        }
+        else
+        {
+            j.m_type = value_t::number_float;
+            j.m_value = val;
+        }
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::number_unsigned>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept
+    {
+        j.m_type = value_t::number_unsigned;
+        j.m_value = val;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::number_integer>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept
+    {
+        j.m_type = value_t::number_integer;
+        j.m_value = val;
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::array>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = arr;
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType, typename CompatibleArrayType,
+             enable_if_t<not std::is_same<CompatibleArrayType,
+                                          typename BasicJsonType::array_t>::value,
+                         int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleArrayType& arr)
+    {
+        using std::begin;
+        using std::end;
+        j.m_type = value_t::array;
+        j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
+        j.assert_invariant();
+    }
+};
+
+template<>
+struct external_constructor<value_t::object>
+{
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj)
+    {
+        j.m_type = value_t::object;
+        j.m_value = obj;
+        j.assert_invariant();
+    }
+
+    template<typename BasicJsonType, typename CompatibleObjectType,
+             enable_if_t<not std::is_same<CompatibleObjectType,
+                                          typename BasicJsonType::object_t>::value,
+                         int> = 0>
+    static void construct(BasicJsonType& j, const CompatibleObjectType& obj)
+    {
+        using std::begin;
+        using std::end;
+
+        j.m_type = value_t::object;
+        j.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj));
+        j.assert_invariant();
+    }
+};
+
+
+////////////////////////
+// has_/is_ functions //
+////////////////////////
+
 /*!
 @brief Helper to determine whether there's a key_type for T.
 
-Thus helper is used to tell associative containers apart from other containers
+This helper is used to tell associative containers apart from other containers
 such as sequence containers. For instance, `std::map` passes the test as it
 contains a `mapped_type`, whereas `std::vector` fails the test.
 
 @sa http://stackoverflow.com/a/7728728/266378
 @since version 1.0.0, overworked in version 2.0.6
 */
-template<typename T>
-struct has_mapped_type
-{
-  private:
-    template <typename U, typename = typename U::mapped_type>
-    static int detect(U&&);
+#define NLOHMANN_JSON_HAS_HELPER(type)                                        \
+    template<typename T> struct has_##type {                                  \
+    private:                                                                  \
+        template<typename U, typename = typename U::type>                     \
+        static int detect(U &&);                                              \
+        static void detect(...);                                              \
+    public:                                                                   \
+        static constexpr bool value =                                         \
+                std::is_integral<decltype(detect(std::declval<T>()))>::value; \
+    }
 
-    static void detect(...);
-  public:
-    static constexpr bool value =
-        std::is_integral<decltype(detect(std::declval<T>()))>::value;
+NLOHMANN_JSON_HAS_HELPER(mapped_type);
+NLOHMANN_JSON_HAS_HELPER(key_type);
+NLOHMANN_JSON_HAS_HELPER(value_type);
+NLOHMANN_JSON_HAS_HELPER(iterator);
+
+#undef NLOHMANN_JSON_HAS_HELPER
+
+
+template<bool B, class RealType, class CompatibleObjectType>
+struct is_compatible_object_type_impl : std::false_type {};
+
+template<class RealType, class CompatibleObjectType>
+struct is_compatible_object_type_impl<true, RealType, CompatibleObjectType>
+{
+    static constexpr auto value =
+        std::is_constructible<typename RealType::key_type,
+        typename CompatibleObjectType::key_type>::value and
+        std::is_constructible<typename RealType::mapped_type,
+        typename CompatibleObjectType::mapped_type>::value;
 };
 
+template<class BasicJsonType, class CompatibleObjectType>
+struct is_compatible_object_type
+{
+    static auto constexpr value = is_compatible_object_type_impl <
+                                  conjunction<negation<std::is_same<void, CompatibleObjectType>>,
+                                  has_mapped_type<CompatibleObjectType>,
+                                  has_key_type<CompatibleObjectType>>::value,
+                                  typename BasicJsonType::object_t, CompatibleObjectType >::value;
+};
+
+template<typename BasicJsonType, typename T>
+struct is_basic_json_nested_type
+{
+    static auto constexpr value = std::is_same<T, typename BasicJsonType::iterator>::value or
+                                  std::is_same<T, typename BasicJsonType::const_iterator>::value or
+                                  std::is_same<T, typename BasicJsonType::reverse_iterator>::value or
+                                  std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value or
+                                  std::is_same<T, typename BasicJsonType::json_pointer>::value;
+};
+
+template<class BasicJsonType, class CompatibleArrayType>
+struct is_compatible_array_type
+{
+    static auto constexpr value =
+        conjunction<negation<std::is_same<void, CompatibleArrayType>>,
+        negation<is_compatible_object_type<
+        BasicJsonType, CompatibleArrayType>>,
+        negation<std::is_constructible<typename BasicJsonType::string_t,
+        CompatibleArrayType>>,
+        negation<is_basic_json_nested_type<BasicJsonType, CompatibleArrayType>>,
+        has_value_type<CompatibleArrayType>,
+        has_iterator<CompatibleArrayType>>::value;
+};
+
+template<bool, typename, typename>
+struct is_compatible_integer_type_impl : std::false_type {};
+
+template<typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type_impl<true, RealIntegerType, CompatibleNumberIntegerType>
+{
+    // is there an assert somewhere on overflows?
+    using RealLimits = std::numeric_limits<RealIntegerType>;
+    using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
+
+    static constexpr auto value =
+        std::is_constructible<RealIntegerType,
+        CompatibleNumberIntegerType>::value and
+        CompatibleLimits::is_integer and
+        RealLimits::is_signed == CompatibleLimits::is_signed;
+};
+
+template<typename RealIntegerType, typename CompatibleNumberIntegerType>
+struct is_compatible_integer_type
+{
+    static constexpr auto value =
+        is_compatible_integer_type_impl <
+        std::is_integral<CompatibleNumberIntegerType>::value and
+        not std::is_same<bool, CompatibleNumberIntegerType>::value,
+        RealIntegerType, CompatibleNumberIntegerType > ::value;
+};
+
+
+// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists
+template<typename BasicJsonType, typename T>
+struct has_from_json
+{
+  private:
+    // also check the return type of from_json
+    template<typename U, typename = enable_if_t<std::is_same<void, decltype(uncvref_t<U>::from_json(
+                 std::declval<BasicJsonType>(), std::declval<T&>()))>::value>>
+    static int detect(U&&);
+    static void detect(...);
+
+  public:
+    static constexpr bool value = std::is_integral<decltype(
+                                      detect(std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
+};
+
+// This trait checks if JSONSerializer<T>::from_json(json const&) exists
+// this overload is used for non-default-constructible user-defined-types
+template<typename BasicJsonType, typename T>
+struct has_non_default_from_json
+{
+  private:
+    template <
+        typename U,
+        typename = enable_if_t<std::is_same<
+                                   T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value >>
+    static int detect(U&&);
+    static void detect(...);
+
+  public:
+    static constexpr bool value = std::is_integral<decltype(detect(
+                                      std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
+};
+
+// This trait checks if BasicJsonType::json_serializer<T>::to_json exists
+template<typename BasicJsonType, typename T>
+struct has_to_json
+{
+  private:
+    template<typename U, typename = decltype(uncvref_t<U>::to_json(
+                 std::declval<BasicJsonType&>(), std::declval<T>()))>
+    static int detect(U&&);
+    static void detect(...);
+
+  public:
+    static constexpr bool value = std::is_integral<decltype(detect(
+                                      std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value;
+};
+
+
+/////////////
+// to_json //
+/////////////
+
+template<typename BasicJsonType, typename T, enable_if_t<
+             std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0>
+void to_json(BasicJsonType& j, T b) noexcept
+{
+    external_constructor<value_t::boolean>::construct(j, b);
 }
 
+template<typename BasicJsonType, typename CompatibleString,
+         enable_if_t<std::is_constructible<typename BasicJsonType::string_t,
+                     CompatibleString>::value, int> = 0>
+void to_json(BasicJsonType& j, const CompatibleString& s)
+{
+    external_constructor<value_t::string>::construct(j, s);
+}
+
+template<typename BasicJsonType, typename FloatType,
+         enable_if_t<std::is_floating_point<FloatType>::value, int> = 0>
+void to_json(BasicJsonType& j, FloatType val) noexcept
+{
+    external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val));
+}
+
+template <
+    typename BasicJsonType, typename CompatibleNumberUnsignedType,
+    enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t,
+                CompatibleNumberUnsignedType>::value, int> = 0 >
+void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept
+{
+    external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val));
+}
+
+template <
+    typename BasicJsonType, typename CompatibleNumberIntegerType,
+    enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t,
+                CompatibleNumberIntegerType>::value, int> = 0 >
+void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept
+{
+    external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));
+}
+
+template<typename BasicJsonType, typename UnscopedEnumType,
+         enable_if_t<is_unscoped_enum<UnscopedEnumType>::value, int> = 0>
+void to_json(BasicJsonType& j, UnscopedEnumType e) noexcept
+{
+    external_constructor<value_t::number_integer>::construct(j, e);
+}
+
+template <
+    typename BasicJsonType, typename CompatibleArrayType,
+    enable_if_t <
+        is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or
+        std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value,
+        int > = 0 >
+void to_json(BasicJsonType& j, const  CompatibleArrayType& arr)
+{
+    external_constructor<value_t::array>::construct(j, arr);
+}
+
+template <
+    typename BasicJsonType, typename CompatibleObjectType,
+    enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value,
+                int> = 0 >
+void to_json(BasicJsonType& j, const  CompatibleObjectType& arr)
+{
+    external_constructor<value_t::object>::construct(j, arr);
+}
+
+
+///////////////
+// from_json //
+///////////////
+
+// overloads for basic_json template parameters
+template<typename BasicJsonType, typename ArithmeticType,
+         enable_if_t<std::is_arithmetic<ArithmeticType>::value and
+                     not std::is_same<ArithmeticType,
+                                      typename BasicJsonType::boolean_t>::value,
+                     int> = 0>
+void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
+{
+    switch (static_cast<value_t>(j))
+    {
+        case value_t::number_unsigned:
+        {
+            val = static_cast<ArithmeticType>(
+                      *j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+            break;
+        }
+        case value_t::number_integer:
+        {
+            val = static_cast<ArithmeticType>(
+                      *j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+            break;
+        }
+        case value_t::number_float:
+        {
+            val = static_cast<ArithmeticType>(
+                      *j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+            break;
+        }
+        default:
+        {
+            JSON_THROW(
+                std::domain_error("type must be number, but is " + j.type_name()));
+        }
+    }
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
+{
+    if (not j.is_boolean())
+    {
+        JSON_THROW(std::domain_error("type must be boolean, but is " + j.type_name()));
+    }
+    b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
+{
+    if (not j.is_string())
+    {
+        JSON_THROW(std::domain_error("type must be string, but is " + j.type_name()));
+    }
+    s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val)
+{
+    get_arithmetic_value(j, val);
+}
+
+template<typename BasicJsonType, typename UnscopedEnumType,
+         enable_if_t<is_unscoped_enum<UnscopedEnumType>::value, int> = 0>
+void from_json(const BasicJsonType& j, UnscopedEnumType& e)
+{
+    typename std::underlying_type<UnscopedEnumType>::type val;
+    get_arithmetic_value(j, val);
+    e = static_cast<UnscopedEnumType>(val);
+}
+
+template<typename BasicJsonType>
+void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr)
+{
+    if (not j.is_array())
+    {
+        JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
+    }
+    arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
+}
+
+// forward_list doesn't have an insert method
+template<typename BasicJsonType, typename T, typename Allocator>
+void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)
+{
+    // do not perform the check when user wants to retrieve jsons
+    // (except when it's null.. ?)
+    if (j.is_null())
+    {
+        JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
+    }
+    if (not std::is_same<T, BasicJsonType>::value)
+    {
+        if (not j.is_array())
+        {
+            JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
+        }
+    }
+    for (auto it = j.rbegin(), end = j.rend(); it != end; ++it)
+    {
+        l.push_front(it->template get<T>());
+    }
+}
+
+template<typename BasicJsonType, typename CompatibleArrayType>
+void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0>)
+{
+    using std::begin;
+    using std::end;
+
+    std::transform(j.begin(), j.end(),
+                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
+    {
+        // get<BasicJsonType>() returns *this, this won't call a from_json
+        // method when value_type is BasicJsonType
+        return i.template get<typename CompatibleArrayType::value_type>();
+    });
+}
+
+template<typename BasicJsonType, typename CompatibleArrayType>
+auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1>)
+-> decltype(
+    arr.reserve(std::declval<typename CompatibleArrayType::size_type>()),
+    void())
+{
+    using std::begin;
+    using std::end;
+
+    arr.reserve(j.size());
+    std::transform(
+        j.begin(), j.end(), std::inserter(arr, end(arr)), [](const BasicJsonType & i)
+    {
+        // get<BasicJsonType>() returns *this, this won't call a from_json
+        // method when value_type is BasicJsonType
+        return i.template get<typename CompatibleArrayType::value_type>();
+    });
+}
+
+template<typename BasicJsonType, typename CompatibleArrayType,
+         enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value and
+                     not std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, int> = 0>
+void from_json(const BasicJsonType& j, CompatibleArrayType& arr)
+{
+    if (j.is_null())
+    {
+        JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
+    }
+
+    // when T == BasicJsonType, do not check if value_t is correct
+    if (not std::is_same<typename CompatibleArrayType::value_type, BasicJsonType>::value)
+    {
+        if (not j.is_array())
+        {
+            JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
+        }
+    }
+    from_json_array_impl(j, arr, priority_tag<1> {});
+}
+
+template<typename BasicJsonType, typename CompatibleObjectType,
+         enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0>
+void from_json(const BasicJsonType& j, CompatibleObjectType& obj)
+{
+    if (not j.is_object())
+    {
+        JSON_THROW(std::domain_error("type must be object, but is " + j.type_name()));
+    }
+
+    auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
+    using std::begin;
+    using std::end;
+    // we could avoid the assignment, but this might require a for loop, which
+    // might be less efficient than the container constructor for some
+    // containers (would it?)
+    obj = CompatibleObjectType(begin(*inner_object), end(*inner_object));
+}
+
+// overload for arithmetic types, not chosen for basic_json template arguments
+// (BooleanType, etc..); note: Is it really necessary to provide explicit
+// overloads for boolean_t etc. in case of a custom BooleanType which is not
+// an arithmetic type?
+template<typename BasicJsonType, typename ArithmeticType,
+         enable_if_t <
+             std::is_arithmetic<ArithmeticType>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and
+             not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value,
+             int> = 0>
+void from_json(const BasicJsonType& j, ArithmeticType& val)
+{
+    switch (static_cast<value_t>(j))
+    {
+        case value_t::number_unsigned:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+            break;
+        }
+        case value_t::number_integer:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+            break;
+        }
+        case value_t::number_float:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+            break;
+        }
+        case value_t::boolean:
+        {
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>());
+            break;
+        }
+        default:
+        {
+            JSON_THROW(std::domain_error("type must be number, but is " + j.type_name()));
+        }
+    }
+}
+
+struct to_json_fn
+{
+  private:
+    template<typename BasicJsonType, typename T>
+    auto call(BasicJsonType& j, T&& val, priority_tag<1>) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
+    -> decltype(to_json(j, std::forward<T>(val)), void())
+    {
+        return to_json(j, std::forward<T>(val));
+    }
+
+    template<typename BasicJsonType, typename T>
+    void call(BasicJsonType&, T&&, priority_tag<0>) const noexcept
+    {
+        static_assert(sizeof(BasicJsonType) == 0,
+                      "could not find to_json() method in T's namespace");
+    }
+
+  public:
+    template<typename BasicJsonType, typename T>
+    void operator()(BasicJsonType& j, T&& val) const
+    noexcept(noexcept(std::declval<to_json_fn>().call(j, std::forward<T>(val), priority_tag<1> {})))
+    {
+        return call(j, std::forward<T>(val), priority_tag<1> {});
+    }
+};
+
+struct from_json_fn
+{
+  private:
+    template<typename BasicJsonType, typename T>
+    auto call(const BasicJsonType& j, T& val, priority_tag<1>) const
+    noexcept(noexcept(from_json(j, val)))
+    -> decltype(from_json(j, val), void())
+    {
+        return from_json(j, val);
+    }
+
+    template<typename BasicJsonType, typename T>
+    void call(const BasicJsonType&, T&, priority_tag<0>) const noexcept
+    {
+        static_assert(sizeof(BasicJsonType) == 0,
+                      "could not find from_json() method in T's namespace");
+    }
+
+  public:
+    template<typename BasicJsonType, typename T>
+    void operator()(const BasicJsonType& j, T& val) const
+    noexcept(noexcept(std::declval<from_json_fn>().call(j, val, priority_tag<1> {})))
+    {
+        return call(j, val, priority_tag<1> {});
+    }
+};
+
+// taken from ranges-v3
+template<typename T>
+struct static_const
+{
+    static constexpr T value{};
+};
+
+template<typename T>
+constexpr T static_const<T>::value;
+} // namespace detail
+
+
+/// namespace to hold default `to_json` / `from_json` functions
+namespace
+{
+constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value;
+constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value;
+}
+
+
+/*!
+@brief default JSONSerializer template argument
+
+This serializer ignores the template arguments and uses ADL
+([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl))
+for serialization.
+*/
+template<typename = void, typename = void>
+struct adl_serializer
+{
+    /*!
+    @brief convert a JSON value to any value type
+
+    This function is usually called by the `get()` function of the
+    @ref basic_json class (either explicit or via conversion operators).
+
+    @param[in] j         JSON value to read from
+    @param[in,out] val  value to write to
+    */
+    template<typename BasicJsonType, typename ValueType>
+    static void from_json(BasicJsonType&& j, ValueType& val) noexcept(
+        noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val)))
+    {
+        ::nlohmann::from_json(std::forward<BasicJsonType>(j), val);
+    }
+
+    /*!
+    @brief convert any value type to a JSON value
+
+    This function is usually called by the constructors of the @ref basic_json
+    class.
+
+    @param[in,out] j  JSON value to write to
+    @param[in] val     value to read from
+    */
+    template<typename BasicJsonType, typename ValueType>
+    static void to_json(BasicJsonType& j, ValueType&& val) noexcept(
+        noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val))))
+    {
+        ::nlohmann::to_json(j, std::forward<ValueType>(val));
+    }
+};
+
+
 /*!
 @brief a class to store JSON values
 
@@ -149,11 +963,14 @@ default; will be used in @ref number_integer_t)
 default; will be used in @ref number_float_t)
 @tparam AllocatorType type of the allocator to use (`std::allocator` by
 default)
+@tparam JSONSerializer the serializer to resolve internal calls to `to_json()`
+and `from_json()` (@ref adl_serializer by default)
 
 @requirement The class satisfies the following concept requirements:
 - Basic
  - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible):
-   JSON values can be default constructed. The result will be a JSON null value.
+   JSON values can be default constructed. The result will be a JSON null
+   value.
  - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible):
    A JSON value can be constructed from an rvalue argument.
  - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible):
@@ -168,8 +985,8 @@ default)
  - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType):
    JSON values have
    [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout):
-   All non-static data members are private and standard layout types, the class
-   has no virtual functions or (virtual) base classes.
+   All non-static data members are private and standard layout types, the
+   class has no virtual functions or (virtual) base classes.
 - Library-wide
  - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable):
    JSON values can be compared with `==`, see @ref
@@ -216,21 +1033,26 @@ template <
     class NumberIntegerType = std::int64_t,
     class NumberUnsignedType = std::uint64_t,
     class NumberFloatType = double,
-    template<typename U> class AllocatorType = std::allocator
+    template<typename U> class AllocatorType = std::allocator,
+    template<typename T, typename SFINAE = void> class JSONSerializer = adl_serializer
     >
 class basic_json
 {
   private:
+    template<detail::value_t> friend struct detail::external_constructor;
     /// workaround type for MSVC
     using basic_json_t = basic_json<ObjectType, ArrayType, StringType,
           BooleanType, NumberIntegerType, NumberUnsignedType, NumberFloatType,
-          AllocatorType>;
+          AllocatorType, JSONSerializer>;
 
   public:
+    using value_t = detail::value_t;
     // forward declarations
     template<typename U> class iter_impl;
     template<typename Base> class json_reverse_iterator;
     class json_pointer;
+    template<typename T, typename SFINAE>
+    using json_serializer = JSONSerializer<T, SFINAE>;
 
     /////////////////////
     // container types //
@@ -282,6 +1104,84 @@ class basic_json
         return allocator_type();
     }
 
+    /*!
+    @brief returns version information on the library
+
+    This function returns a JSON object with information about the library,
+    including the version number and information on the platform and compiler.
+
+    @return JSON object holding version information
+    key         | description
+    ----------- | ---------------
+    `compiler`  | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version).
+    `copyright` | The copyright line for the library as string.
+    `name`      | The name of the library as string.
+    `platform`  | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`.
+    `url`       | The URL of the project as string.
+    `version`   | The version of the library. It is an object with the following keys: `major`, `minor`, and `patch` as defined by [Semantic Versioning](http://semver.org), and `string` (the version string).
+
+    @liveexample{The following code shows an example output of the `meta()`
+    function.,meta}
+
+    @complexity Constant.
+
+    @since 2.1.0
+    */
+    static basic_json meta()
+    {
+        basic_json result;
+
+        result["copyright"] = "(C) 2013-2017 Niels Lohmann";
+        result["name"] = "JSON for Modern C++";
+        result["url"] = "https://github.com/nlohmann/json";
+        result["version"] =
+        {
+            {"string", "2.1.1"},
+            {"major", 2},
+            {"minor", 1},
+            {"patch", 1}
+        };
+
+#ifdef _WIN32
+        result["platform"] = "win32";
+#elif defined __linux__
+        result["platform"] = "linux";
+#elif defined __APPLE__
+        result["platform"] = "apple";
+#elif defined __unix__
+        result["platform"] = "unix";
+#else
+        result["platform"] = "unknown";
+#endif
+
+#if defined(__clang__)
+        result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}};
+#elif defined(__ICC) || defined(__INTEL_COMPILER)
+        result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}};
+#elif defined(__GNUC__) || defined(__GNUG__)
+        result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}};
+#elif defined(__HP_cc) || defined(__HP_aCC)
+        result["compiler"] = "hp"
+#elif defined(__IBMCPP__)
+        result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}};
+#elif defined(_MSC_VER)
+        result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}};
+#elif defined(__PGI)
+        result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}};
+#elif defined(__SUNPRO_CC)
+        result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}};
+#else
+        result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}};
+#endif
+
+#ifdef __cplusplus
+        result["compiler"]["c++"] = std::to_string(__cplusplus);
+#else
+        result["compiler"]["c++"] = "unknown";
+#endif
+        return result;
+    }
+
 
     ///////////////////////////
     // JSON value data types //
@@ -449,6 +1349,12 @@ class basic_json
     std::string
     @endcode
 
+    #### Encoding
+
+    Strings are stored in UTF-8 encoding. Therefore, functions like
+    `std::string::size()` or `std::string::length()` return the number of
+    bytes in the string rather than the number of characters or glyphs.
+
     #### String comparison
 
     [RFC 7159](http://rfc7159.net/rfc7159) states:
@@ -713,47 +1619,6 @@ class basic_json
 
     /// @}
 
-
-    ///////////////////////////
-    // JSON type enumeration //
-    ///////////////////////////
-
-    /*!
-    @brief the JSON type enumeration
-
-    This enumeration collects the different JSON types. It is internally used
-    to distinguish the stored values, and the functions @ref is_null(), @ref
-    is_object(), @ref is_array(), @ref is_string(), @ref is_boolean(), @ref
-    is_number() (with @ref is_number_integer(), @ref is_number_unsigned(), and
-    @ref is_number_float()), @ref is_discarded(), @ref is_primitive(), and
-    @ref is_structured() rely on it.
-
-    @note There are three enumeration entries (number_integer,
-    number_unsigned, and number_float), because the library distinguishes
-    these three types for numbers: @ref number_unsigned_t is used for unsigned
-    integers, @ref number_integer_t is used for signed integers, and @ref
-    number_float_t is used for floating-point numbers or to approximate
-    integers which do not fit in the limits of their respective type.
-
-    @sa @ref basic_json(const value_t value_type) -- create a JSON value with
-    the default value for a given type
-
-    @since version 1.0.0
-    */
-    enum class value_t : uint8_t
-    {
-        null,            ///< null value
-        object,          ///< object (unordered set of name/value pairs)
-        array,           ///< array (ordered collection of values)
-        string,          ///< string value
-        boolean,         ///< boolean value
-        number_integer,  ///< number value (signed integer)
-        number_unsigned, ///< number value (unsigned integer)
-        number_float,    ///< number value (floating-point)
-        discarded        ///< discarded by the the parser callback function
-    };
-
-
   private:
 
     /// helper for exception-safe object creation
@@ -767,7 +1632,7 @@ class basic_json
         };
         std::unique_ptr<T, decltype(deleter)> object(alloc.allocate(1), deleter);
         alloc.construct(object.get(), std::forward<Args>(args)...);
-        assert(object.get() != nullptr);
+        assert(object != nullptr);
         return object.release();
     }
 
@@ -882,7 +1747,7 @@ class basic_json
                 {
                     if (t == value_t::null)
                     {
-                        throw std::domain_error("961c151d2e87f2686a955a9be24d316f1362bf21 2.0.10"); // LCOV_EXCL_LINE
+                        JSON_THROW(std::domain_error("961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE
                     }
                     break;
                 }
@@ -1046,18 +1911,6 @@ class basic_json
     @liveexample{The following code shows the constructor for different @ref
     value_t values,basic_json__value_t}
 
-    @sa @ref basic_json(std::nullptr_t) -- create a `null` value
-    @sa @ref basic_json(boolean_t value) -- create a boolean value
-    @sa @ref basic_json(const string_t&) -- create a string value
-    @sa @ref basic_json(const object_t&) -- create a object value
-    @sa @ref basic_json(const array_t&) -- create a array value
-    @sa @ref basic_json(const number_float_t) -- create a number
-    (floating-point) value
-    @sa @ref basic_json(const number_integer_t) -- create a number (integer)
-    value
-    @sa @ref basic_json(const number_unsigned_t) -- create a number (unsigned)
-    value
-
     @since version 1.0.0
     */
     basic_json(const value_t value_type)
@@ -1091,474 +1944,69 @@ class basic_json
     }
 
     /*!
-    @brief create an object (explicit)
+    @brief create a JSON value
 
-    Create an object JSON value with a given content.
+    This is a "catch all" constructor for all compatible JSON types; that is,
+    types for which a `to_json()` method exsits. The constructor forwards the
+    parameter @a val to that method (to `json_serializer<U>::to_json` method
+    with `U = uncvref_t<CompatibleType>`, to be exact).
 
-    @param[in] val  a value for the object
+    Template type @a CompatibleType includes, but is not limited to, the
+    following types:
+    - **arrays**: @ref array_t and all kinds of compatible containers such as
+      `std::vector`, `std::deque`, `std::list`, `std::forward_list`,
+      `std::array`, `std::set`, `std::unordered_set`, `std::multiset`, and
+      `unordered_multiset` with a `value_type` from which a @ref basic_json
+      value can be constructed.
+    - **objects**: @ref object_t and all kinds of compatible associative
+      containers such as `std::map`, `std::unordered_map`, `std::multimap`,
+      and `std::unordered_multimap` with a `key_type` compatible to
+      @ref string_t and a `value_type` from which a @ref basic_json value can
+      be constructed.
+    - **strings**: @ref string_t, string literals, and all compatible string
+      containers can be used.
+    - **numbers**: @ref number_integer_t, @ref number_unsigned_t,
+      @ref number_float_t, and all convertible number types such as `int`,
+      `size_t`, `int64_t`, `float` or `double` can be used.
+    - **boolean**: @ref boolean_t / `bool` can be used.
 
-    @complexity Linear in the size of the passed @a val.
+    See the examples below.
 
-    @throw std::bad_alloc if allocation for object value fails
+    @tparam CompatibleType a type such that:
+    - @a CompatibleType is not derived from `std::istream`,
+    - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move
+         constructors),
+    - @a CompatibleType is not a @ref basic_json nested type (e.g.,
+         @ref json_pointer, @ref iterator, etc ...)
+    - @ref @ref json_serializer<U> has a
+         `to_json(basic_json_t&, CompatibleType&&)` method
 
-    @liveexample{The following code shows the constructor with an @ref
-    object_t parameter.,basic_json__object_t}
+    @tparam U = `uncvref_t<CompatibleType>`
 
-    @sa @ref basic_json(const CompatibleObjectType&) -- create an object value
-    from a compatible STL container
+    @param[in] val the value to be forwarded
 
-    @since version 1.0.0
-    */
-    basic_json(const object_t& val)
-        : m_type(value_t::object), m_value(val)
-    {
-        assert_invariant();
-    }
+    @complexity Usually linear in the size of the passed @a val, also
+                depending on the implementation of the called `to_json()`
+                method.
 
-    /*!
-    @brief create an object (implicit)
-
-    Create an object JSON value with a given content. This constructor allows
-    any type @a CompatibleObjectType that can be used to construct values of
-    type @ref object_t.
-
-    @tparam CompatibleObjectType An object type whose `key_type` and
-    `value_type` is compatible to @ref object_t. Examples include `std::map`,
-    `std::unordered_map`, `std::multimap`, and `std::unordered_multimap` with
-    a `key_type` of `std::string`, and a `value_type` from which a @ref
-    basic_json value can be constructed.
-
-    @param[in] val  a value for the object
-
-    @complexity Linear in the size of the passed @a val.
-
-    @throw std::bad_alloc if allocation for object value fails
+    @throw what `json_serializer<U>::to_json()` throws
 
     @liveexample{The following code shows the constructor with several
-    compatible object type parameters.,basic_json__CompatibleObjectType}
+    compatible types.,basic_json__CompatibleType}
 
-    @sa @ref basic_json(const object_t&) -- create an object value
-
-    @since version 1.0.0
+    @since version 2.1.0
     */
-    template<class CompatibleObjectType, typename std::enable_if<
-                 std::is_constructible<typename object_t::key_type, typename CompatibleObjectType::key_type>::value and
-                 std::is_constructible<basic_json, typename CompatibleObjectType::mapped_type>::value, int>::type = 0>
-    basic_json(const CompatibleObjectType& val)
-        : m_type(value_t::object)
-    {
-        using std::begin;
-        using std::end;
-        m_value.object = create<object_t>(begin(val), end(val));
-        assert_invariant();
-    }
-
-    /*!
-    @brief create an array (explicit)
-
-    Create an array JSON value with a given content.
-
-    @param[in] val  a value for the array
-
-    @complexity Linear in the size of the passed @a val.
-
-    @throw std::bad_alloc if allocation for array value fails
-
-    @liveexample{The following code shows the constructor with an @ref array_t
-    parameter.,basic_json__array_t}
-
-    @sa @ref basic_json(const CompatibleArrayType&) -- create an array value
-    from a compatible STL containers
-
-    @since version 1.0.0
-    */
-    basic_json(const array_t& val)
-        : m_type(value_t::array), m_value(val)
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create an array (implicit)
-
-    Create an array JSON value with a given content. This constructor allows
-    any type @a CompatibleArrayType that can be used to construct values of
-    type @ref array_t.
-
-    @tparam CompatibleArrayType An object type whose `value_type` is
-    compatible to @ref array_t. Examples include `std::vector`, `std::deque`,
-    `std::list`, `std::forward_list`, `std::array`, `std::set`,
-    `std::unordered_set`, `std::multiset`, and `unordered_multiset` with a
-    `value_type` from which a @ref basic_json value can be constructed.
-
-    @param[in] val  a value for the array
-
-    @complexity Linear in the size of the passed @a val.
-
-    @throw std::bad_alloc if allocation for array value fails
-
-    @liveexample{The following code shows the constructor with several
-    compatible array type parameters.,basic_json__CompatibleArrayType}
-
-    @sa @ref basic_json(const array_t&) -- create an array value
-
-    @since version 1.0.0
-    */
-    template<class CompatibleArrayType, typename std::enable_if<
-                 not std::is_same<CompatibleArrayType, typename basic_json_t::iterator>::value and
-                 not std::is_same<CompatibleArrayType, typename basic_json_t::const_iterator>::value and
-                 not std::is_same<CompatibleArrayType, typename basic_json_t::reverse_iterator>::value and
-                 not std::is_same<CompatibleArrayType, typename basic_json_t::const_reverse_iterator>::value and
-                 not std::is_same<CompatibleArrayType, typename array_t::iterator>::value and
-                 not std::is_same<CompatibleArrayType, typename array_t::const_iterator>::value and
-                 std::is_constructible<basic_json, typename CompatibleArrayType::value_type>::value, int>::type = 0>
-    basic_json(const CompatibleArrayType& val)
-        : m_type(value_t::array)
-    {
-        using std::begin;
-        using std::end;
-        m_value.array = create<array_t>(begin(val), end(val));
-        assert_invariant();
-    }
-
-    /*!
-    @brief create a string (explicit)
-
-    Create an string JSON value with a given content.
-
-    @param[in] val  a value for the string
-
-    @complexity Linear in the size of the passed @a val.
-
-    @throw std::bad_alloc if allocation for string value fails
-
-    @liveexample{The following code shows the constructor with an @ref
-    string_t parameter.,basic_json__string_t}
-
-    @sa @ref basic_json(const typename string_t::value_type*) -- create a
-    string value from a character pointer
-    @sa @ref basic_json(const CompatibleStringType&) -- create a string value
-    from a compatible string container
-
-    @since version 1.0.0
-    */
-    basic_json(const string_t& val)
-        : m_type(value_t::string), m_value(val)
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create a string (explicit)
-
-    Create a string JSON value with a given content.
-
-    @param[in] val  a literal value for the string
-
-    @complexity Linear in the size of the passed @a val.
-
-    @throw std::bad_alloc if allocation for string value fails
-
-    @liveexample{The following code shows the constructor with string literal
-    parameter.,basic_json__string_t_value_type}
-
-    @sa @ref basic_json(const string_t&) -- create a string value
-    @sa @ref basic_json(const CompatibleStringType&) -- create a string value
-    from a compatible string container
-
-    @since version 1.0.0
-    */
-    basic_json(const typename string_t::value_type* val)
-        : basic_json(string_t(val))
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create a string (implicit)
-
-    Create a string JSON value with a given content.
-
-    @param[in] val  a value for the string
-
-    @tparam CompatibleStringType an string type which is compatible to @ref
-    string_t, for instance `std::string`.
-
-    @complexity Linear in the size of the passed @a val.
-
-    @throw std::bad_alloc if allocation for string value fails
-
-    @liveexample{The following code shows the construction of a string value
-    from a compatible type.,basic_json__CompatibleStringType}
-
-    @sa @ref basic_json(const string_t&) -- create a string value
-    @sa @ref basic_json(const typename string_t::value_type*) -- create a
-    string value from a character pointer
-
-    @since version 1.0.0
-    */
-    template<class CompatibleStringType, typename std::enable_if<
-                 std::is_constructible<string_t, CompatibleStringType>::value, int>::type = 0>
-    basic_json(const CompatibleStringType& val)
-        : basic_json(string_t(val))
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create a boolean (explicit)
-
-    Creates a JSON boolean type from a given value.
-
-    @param[in] val  a boolean value to store
-
-    @complexity Constant.
-
-    @liveexample{The example below demonstrates boolean
-    values.,basic_json__boolean_t}
-
-    @since version 1.0.0
-    */
-    basic_json(boolean_t val) noexcept
-        : m_type(value_t::boolean), m_value(val)
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create an integer number (explicit)
-
-    Create an integer number JSON value with a given content.
-
-    @tparam T A helper type to remove this function via SFINAE in case @ref
-    number_integer_t is the same as `int`. In this case, this constructor
-    would have the same signature as @ref basic_json(const int value). Note
-    the helper type @a T is not visible in this constructor's interface.
-
-    @param[in] val  an integer to create a JSON number from
-
-    @complexity Constant.
-
-    @liveexample{The example below shows the construction of an integer
-    number value.,basic_json__number_integer_t}
-
-    @sa @ref basic_json(const int) -- create a number value (integer)
-    @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number
-    value (integer) from a compatible number type
-
-    @since version 1.0.0
-    */
-    template<typename T, typename std::enable_if<
-                 not (std::is_same<T, int>::value) and
-                 std::is_same<T, number_integer_t>::value, int>::type = 0>
-    basic_json(const number_integer_t val) noexcept
-        : m_type(value_t::number_integer), m_value(val)
-    {
-        assert_invariant();
-    }
-  
-  
-    /*!
-    @brief create an integer number from an enum type (explicit)
-
-    Create an integer number JSON value with a given content.
-
-    @param[in] val  an integer to create a JSON number from
-
-    @note This constructor allows to pass enums directly to a constructor. As
-    C++ has no way of specifying the type of an anonymous enum explicitly, we
-    can only rely on the fact that such values implicitly convert to int. As
-    int may already be the same type of number_integer_t, we may need to
-    switch off the constructor @ref basic_json(const number_integer_t).
-
-    @complexity Constant.
-
-    @liveexample{The example below shows the construction of an integer
-    number value from an anonymous enum.,basic_json__const_int}
-
-    @sa @ref basic_json(const number_integer_t) -- create a number value
-    (integer)
-    @sa @ref basic_json(const CompatibleNumberIntegerType) -- create a number
-    value (integer) from a compatible number type
-
-    @since version 1.0.0
-    */
-    basic_json(const int val) noexcept
-        : m_type(value_t::number_integer),
-          m_value(static_cast<number_integer_t>(val))
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create an integer number (implicit)
-
-    Create an integer number JSON value with a given content. This constructor
-    allows any type @a CompatibleNumberIntegerType that can be used to
-    construct values of type @ref number_integer_t.
-
-    @tparam CompatibleNumberIntegerType An integer type which is compatible to
-    @ref number_integer_t. Examples include the types `int`, `int32_t`,
-    `long`, and `short`.
-
-    @param[in] val  an integer to create a JSON number from
-
-    @complexity Constant.
-
-    @liveexample{The example below shows the construction of several integer
-    number values from compatible
-    types.,basic_json__CompatibleIntegerNumberType}
-
-    @sa @ref basic_json(const number_integer_t) -- create a number value
-    (integer)
-    @sa @ref basic_json(const int) -- create a number value (integer)
-
-    @since version 1.0.0
-    */
-    template<typename CompatibleNumberIntegerType, typename std::enable_if<
-                 std::is_constructible<number_integer_t, CompatibleNumberIntegerType>::value and
-                 std::numeric_limits<CompatibleNumberIntegerType>::is_integer and
-                 std::numeric_limits<CompatibleNumberIntegerType>::is_signed,
-                 CompatibleNumberIntegerType>::type = 0>
-    basic_json(const CompatibleNumberIntegerType val) noexcept
-        : m_type(value_t::number_integer),
-          m_value(static_cast<number_integer_t>(val))
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create an unsigned integer number (explicit)
-
-    Create an unsigned integer number JSON value with a given content.
-
-    @tparam T  helper type to compare number_unsigned_t and unsigned int (not
-    visible in) the interface.
-
-    @param[in] val  an integer to create a JSON number from
-
-    @complexity Constant.
-
-    @sa @ref basic_json(const CompatibleNumberUnsignedType) -- create a number
-    value (unsigned integer) from a compatible number type
-
-    @since version 2.0.0
-    */
-    template<typename T, typename std::enable_if<
-                 not (std::is_same<T, int>::value) and
-                 std::is_same<T, number_unsigned_t>::value, int>::type = 0>
-    basic_json(const number_unsigned_t val) noexcept
-        : m_type(value_t::number_unsigned), m_value(val)
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create an unsigned number (implicit)
-
-    Create an unsigned number JSON value with a given content. This
-    constructor allows any type @a CompatibleNumberUnsignedType that can be
-    used to construct values of type @ref number_unsigned_t.
-
-    @tparam CompatibleNumberUnsignedType An integer type which is compatible
-    to @ref number_unsigned_t. Examples may include the types `unsigned int`,
-    `uint32_t`, or `unsigned short`.
-
-    @param[in] val  an unsigned integer to create a JSON number from
-
-    @complexity Constant.
-
-    @sa @ref basic_json(const number_unsigned_t) -- create a number value
-    (unsigned)
-
-    @since version 2.0.0
-    */
-    template<typename CompatibleNumberUnsignedType, typename std::enable_if <
-                 std::is_constructible<number_unsigned_t, CompatibleNumberUnsignedType>::value and
-                 std::numeric_limits<CompatibleNumberUnsignedType>::is_integer and
-                 not std::numeric_limits<CompatibleNumberUnsignedType>::is_signed,
-                 CompatibleNumberUnsignedType>::type = 0>
-    basic_json(const CompatibleNumberUnsignedType val) noexcept
-        : m_type(value_t::number_unsigned),
-          m_value(static_cast<number_unsigned_t>(val))
-    {
-        assert_invariant();
-    }
-
-    /*!
-    @brief create a floating-point number (explicit)
-
-    Create a floating-point number JSON value with a given content.
-
-    @param[in] val  a floating-point value to create a JSON number from
-
-    @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6
-    disallows NaN values:
-    > Numeric values that cannot be represented in the grammar below (such as
-    > Infinity and NaN) are not permitted.
-    In case the parameter @a val is not a number, a JSON null value is created
-    instead.
-
-    @complexity Constant.
-
-    @liveexample{The following example creates several floating-point
-    values.,basic_json__number_float_t}
-
-    @sa @ref basic_json(const CompatibleNumberFloatType) -- create a number
-    value (floating-point) from a compatible number type
-
-    @since version 1.0.0
-    */
-    basic_json(const number_float_t val) noexcept
-        : m_type(value_t::number_float), m_value(val)
-    {
-        // replace infinity and NAN by null
-        if (not std::isfinite(val))
-        {
-            m_type = value_t::null;
-            m_value = json_value();
-        }
-
-        assert_invariant();
-    }
-
-    /*!
-    @brief create an floating-point number (implicit)
-
-    Create an floating-point number JSON value with a given content. This
-    constructor allows any type @a CompatibleNumberFloatType that can be used
-    to construct values of type @ref number_float_t.
-
-    @tparam CompatibleNumberFloatType A floating-point type which is
-    compatible to @ref number_float_t. Examples may include the types `float`
-    or `double`.
-
-    @param[in] val  a floating-point to create a JSON number from
-
-    @note [RFC 7159](http://www.rfc-editor.org/rfc/rfc7159.txt), section 6
-    disallows NaN values:
-    > Numeric values that cannot be represented in the grammar below (such as
-    > Infinity and NaN) are not permitted.
-    In case the parameter @a val is not a number, a JSON null value is
-    created instead.
-
-    @complexity Constant.
-
-    @liveexample{The example below shows the construction of several
-    floating-point number values from compatible
-    types.,basic_json__CompatibleNumberFloatType}
-
-    @sa @ref basic_json(const number_float_t) -- create a number value
-    (floating-point)
-
-    @since version 1.0.0
-    */
-    template<typename CompatibleNumberFloatType, typename = typename std::enable_if<
-                 std::is_constructible<number_float_t, CompatibleNumberFloatType>::value and
-                 std::is_floating_point<CompatibleNumberFloatType>::value>::type>
-    basic_json(const CompatibleNumberFloatType val) noexcept
-        : basic_json(number_float_t(val))
+    template<typename CompatibleType, typename U = detail::uncvref_t<CompatibleType>,
+             detail::enable_if_t<not std::is_base_of<std::istream, U>::value and
+                                 not std::is_same<U, basic_json_t>::value and
+                                 not detail::is_basic_json_nested_type<
+                                     basic_json_t, U>::value and
+                                 detail::has_to_json<basic_json, U>::value,
+                                 int> = 0>
+    basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer<U>::to_json(
+                std::declval<basic_json_t&>(), std::forward<CompatibleType>(val))))
     {
+        JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val));
         assert_invariant();
     }
 
@@ -1655,7 +2103,7 @@ class basic_json
             // if object is wanted but impossible, throw an exception
             if (manual_type == value_t::object and not is_an_object)
             {
-                throw std::domain_error("cannot create object from initializer list");
+                JSON_THROW(std::domain_error("cannot create object from initializer list"));
             }
         }
 
@@ -1833,7 +2281,7 @@ class basic_json
         // make sure iterator fits the current value
         if (first.m_object != last.m_object)
         {
-            throw std::domain_error("iterators are not compatible");
+            JSON_THROW(std::domain_error("iterators are not compatible"));
         }
 
         // copy type from first iterator
@@ -1850,7 +2298,7 @@ class basic_json
             {
                 if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end())
                 {
-                    throw std::out_of_range("iterators out of range");
+                    JSON_THROW(std::out_of_range("iterators out of range"));
                 }
                 break;
             }
@@ -1895,19 +2343,21 @@ class basic_json
 
             case value_t::object:
             {
-                m_value.object = create<object_t>(first.m_it.object_iterator, last.m_it.object_iterator);
+                m_value.object = create<object_t>(first.m_it.object_iterator,
+                                                  last.m_it.object_iterator);
                 break;
             }
 
             case value_t::array:
             {
-                m_value.array = create<array_t>(first.m_it.array_iterator, last.m_it.array_iterator);
+                m_value.array = create<array_t>(first.m_it.array_iterator,
+                                                last.m_it.array_iterator);
                 break;
             }
 
             default:
             {
-                throw std::domain_error("cannot use construct with iterators from " + first.m_object->type_name());
+                JSON_THROW(std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()));
             }
         }
 
@@ -2197,14 +2647,6 @@ class basic_json
     string_t dump(const int indent = -1) const
     {
         std::stringstream ss;
-        // fix locale problems
-        ss.imbue(std::locale::classic());
-
-        // 6, 15 or 16 digits of precision allows round-trip IEEE 754
-        // string->float->string, string->double->string or string->long
-        // double->string; to be safe, we read this value from
-        // std::numeric_limits<number_float_t>::digits10
-        ss.precision(std::numeric_limits<double>::digits10);
 
         if (indent >= 0)
         {
@@ -2580,244 +3022,97 @@ class basic_json
     // value access //
     //////////////////
 
-    /// get an object (explicit)
-    template<class T, typename std::enable_if<
-                 std::is_convertible<typename object_t::key_type, typename T::key_type>::value and
-                 std::is_convertible<basic_json_t, typename T::mapped_type>::value, int>::type = 0>
-    T get_impl(T*) const
-    {
-        if (is_object())
-        {
-            return T(m_value.object->begin(), m_value.object->end());
-        }
-        else
-        {
-            throw std::domain_error("type must be object, but is " + type_name());
-        }
-    }
-
-    /// get an object (explicit)
-    object_t get_impl(object_t*) const
-    {
-        if (is_object())
-        {
-            return *(m_value.object);
-        }
-        else
-        {
-            throw std::domain_error("type must be object, but is " + type_name());
-        }
-    }
-
-    /// get an array (explicit)
-    template<class T, typename std::enable_if<
-                 std::is_convertible<basic_json_t, typename T::value_type>::value and
-                 not std::is_same<basic_json_t, typename T::value_type>::value and
-                 not std::is_arithmetic<T>::value and
-                 not std::is_convertible<std::string, T>::value and
-                 not has_mapped_type<T>::value, int>::type = 0>
-    T get_impl(T*) const
-    {
-        if (is_array())
-        {
-            T to_vector;
-            std::transform(m_value.array->begin(), m_value.array->end(),
-                           std::inserter(to_vector, to_vector.end()), [](basic_json i)
-            {
-                return i.get<typename T::value_type>();
-            });
-            return to_vector;
-        }
-        else
-        {
-            throw std::domain_error("type must be array, but is " + type_name());
-        }
-    }
-
-    /// get an array (explicit)
-    template<class T, typename std::enable_if<
-                 std::is_convertible<basic_json_t, T>::value and
-                 not std::is_same<basic_json_t, T>::value, int>::type = 0>
-    std::vector<T> get_impl(std::vector<T>*) const
-    {
-        if (is_array())
-        {
-            std::vector<T> to_vector;
-            to_vector.reserve(m_value.array->size());
-            std::transform(m_value.array->begin(), m_value.array->end(),
-                           std::inserter(to_vector, to_vector.end()), [](basic_json i)
-            {
-                return i.get<T>();
-            });
-            return to_vector;
-        }
-        else
-        {
-            throw std::domain_error("type must be array, but is " + type_name());
-        }
-    }
-
-    /// get an array (explicit)
-    template<class T, typename std::enable_if<
-                 std::is_same<basic_json, typename T::value_type>::value and
-                 not has_mapped_type<T>::value, int>::type = 0>
-    T get_impl(T*) const
-    {
-        if (is_array())
-        {
-            return T(m_value.array->begin(), m_value.array->end());
-        }
-        else
-        {
-            throw std::domain_error("type must be array, but is " + type_name());
-        }
-    }
-
-    /// get an array (explicit)
-    array_t get_impl(array_t*) const
-    {
-        if (is_array())
-        {
-            return *(m_value.array);
-        }
-        else
-        {
-            throw std::domain_error("type must be array, but is " + type_name());
-        }
-    }
-
-    /// get a string (explicit)
-    template<typename T, typename std::enable_if<
-                 std::is_convertible<string_t, T>::value, int>::type = 0>
-    T get_impl(T*) const
-    {
-        if (is_string())
-        {
-            return *m_value.string;
-        }
-        else
-        {
-            throw std::domain_error("type must be string, but is " + type_name());
-        }
-    }
-
-    /// get a number (explicit)
-    template<typename T, typename std::enable_if<
-                 std::is_arithmetic<T>::value, int>::type = 0>
-    T get_impl(T*) const
-    {
-        switch (m_type)
-        {
-            case value_t::number_integer:
-            {
-                return static_cast<T>(m_value.number_integer);
-            }
-
-            case value_t::number_unsigned:
-            {
-                return static_cast<T>(m_value.number_unsigned);
-            }
-
-            case value_t::number_float:
-            {
-                return static_cast<T>(m_value.number_float);
-            }
-
-            default:
-            {
-                throw std::domain_error("type must be number, but is " + type_name());
-            }
-        }
-    }
-
     /// get a boolean (explicit)
-    constexpr boolean_t get_impl(boolean_t*) const
+    boolean_t get_impl(boolean_t* /*unused*/) const
     {
-        return is_boolean()
-               ? m_value.boolean
-               : throw std::domain_error("type must be boolean, but is " + type_name());
+        if (is_boolean())
+        {
+            return m_value.boolean;
+        }
+
+        JSON_THROW(std::domain_error("type must be boolean, but is " + type_name()));
     }
 
     /// get a pointer to the value (object)
-    object_t* get_impl_ptr(object_t*) noexcept
+    object_t* get_impl_ptr(object_t* /*unused*/) noexcept
     {
         return is_object() ? m_value.object : nullptr;
     }
 
     /// get a pointer to the value (object)
-    constexpr const object_t* get_impl_ptr(const object_t*) const noexcept
+    constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept
     {
         return is_object() ? m_value.object : nullptr;
     }
 
     /// get a pointer to the value (array)
-    array_t* get_impl_ptr(array_t*) noexcept
+    array_t* get_impl_ptr(array_t* /*unused*/) noexcept
     {
         return is_array() ? m_value.array : nullptr;
     }
 
     /// get a pointer to the value (array)
-    constexpr const array_t* get_impl_ptr(const array_t*) const noexcept
+    constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept
     {
         return is_array() ? m_value.array : nullptr;
     }
 
     /// get a pointer to the value (string)
-    string_t* get_impl_ptr(string_t*) noexcept
+    string_t* get_impl_ptr(string_t* /*unused*/) noexcept
     {
         return is_string() ? m_value.string : nullptr;
     }
 
     /// get a pointer to the value (string)
-    constexpr const string_t* get_impl_ptr(const string_t*) const noexcept
+    constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept
     {
         return is_string() ? m_value.string : nullptr;
     }
 
     /// get a pointer to the value (boolean)
-    boolean_t* get_impl_ptr(boolean_t*) noexcept
+    boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept
     {
         return is_boolean() ? &m_value.boolean : nullptr;
     }
 
     /// get a pointer to the value (boolean)
-    constexpr const boolean_t* get_impl_ptr(const boolean_t*) const noexcept
+    constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept
     {
         return is_boolean() ? &m_value.boolean : nullptr;
     }
 
     /// get a pointer to the value (integer number)
-    number_integer_t* get_impl_ptr(number_integer_t*) noexcept
+    number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept
     {
         return is_number_integer() ? &m_value.number_integer : nullptr;
     }
 
     /// get a pointer to the value (integer number)
-    constexpr const number_integer_t* get_impl_ptr(const number_integer_t*) const noexcept
+    constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept
     {
         return is_number_integer() ? &m_value.number_integer : nullptr;
     }
 
     /// get a pointer to the value (unsigned number)
-    number_unsigned_t* get_impl_ptr(number_unsigned_t*) noexcept
+    number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept
     {
         return is_number_unsigned() ? &m_value.number_unsigned : nullptr;
     }
 
     /// get a pointer to the value (unsigned number)
-    constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t*) const noexcept
+    constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept
     {
         return is_number_unsigned() ? &m_value.number_unsigned : nullptr;
     }
 
     /// get a pointer to the value (floating-point number)
-    number_float_t* get_impl_ptr(number_float_t*) noexcept
+    number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept
     {
         return is_number_float() ? &m_value.number_float : nullptr;
     }
 
     /// get a pointer to the value (floating-point number)
-    constexpr const number_float_t* get_impl_ptr(const number_float_t*) const noexcept
+    constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept
     {
         return is_number_float() ? &m_value.number_float : nullptr;
     }
@@ -2846,34 +3141,69 @@ class basic_json
         {
             return *ptr;
         }
-        else
-        {
-            throw std::domain_error("incompatible ReferenceType for get_ref, actual type is " +
-                                    obj.type_name());
-        }
+
+        JSON_THROW(std::domain_error("incompatible ReferenceType for get_ref, actual type is " +
+                                     obj.type_name()));
     }
 
   public:
-
     /// @name value access
     /// Direct access to the stored value of a JSON value.
     /// @{
 
+    /*!
+    @brief get special-case overload
+
+    This overloads avoids a lot of template boilerplate, it can be seen as the
+    identity method
+
+    @tparam BasicJsonType == @ref basic_json
+
+    @return a copy of *this
+
+    @complexity Constant.
+
+    @since version 2.1.0
+    */
+    template <
+        typename BasicJsonType,
+        detail::enable_if_t<std::is_same<typename std::remove_const<BasicJsonType>::type,
+                                         basic_json_t>::value,
+                            int> = 0 >
+    basic_json get() const
+    {
+        return *this;
+    }
+
     /*!
     @brief get a value (explicit)
 
-    Explicit type conversion between the JSON value and a compatible value.
+    Explicit type conversion between the JSON value and a compatible value
+    which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible)
+    and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible).
+    The value is converted by calling the @ref json_serializer<ValueType>
+    `from_json()` method.
 
-    @tparam ValueType non-pointer type compatible to the JSON value, for
-    instance `int` for JSON integer numbers, `bool` for JSON booleans, or
-    `std::vector` types for JSON arrays
+    The function is equivalent to executing
+    @code {.cpp}
+    ValueType ret;
+    JSONSerializer<ValueType>::from_json(*this, ret);
+    return ret;
+    @endcode
 
-    @return copy of the JSON value, converted to type @a ValueType
+    This overloads is chosen if:
+    - @a ValueType is not @ref basic_json,
+    - @ref json_serializer<ValueType> has a `from_json()` method of the form
+      `void from_json(const @ref basic_json&, ValueType&)`, and
+    - @ref json_serializer<ValueType> does not have a `from_json()` method of
+      the form `ValueType from_json(const @ref basic_json&)`
 
-    @throw std::domain_error in case passed type @a ValueType is incompatible
-    to JSON; example: `"type must be object, but is null"`
+    @tparam ValueTypeCV the provided value type
+    @tparam ValueType the returned value type
 
-    @complexity Linear in the size of the JSON value.
+    @return copy of the JSON value, converted to @a ValueType
+
+    @throw what @ref json_serializer<ValueType> `from_json()` method throws
 
     @liveexample{The example below shows several conversions from JSON values
     to other types. There a few things to note: (1) Floating-point numbers can
@@ -2882,21 +3212,75 @@ class basic_json
     associative containers such as `std::unordered_map<std::string\,
     json>`.,get__ValueType_const}
 
-    @internal
-    The idea of using a casted null pointer to choose the correct
-    implementation is from <http://stackoverflow.com/a/8315197/266378>.
-    @endinternal
-
-    @sa @ref operator ValueType() const for implicit conversion
-    @sa @ref get() for pointer-member access
-
-    @since version 1.0.0
+    @since version 2.1.0
     */
-    template<typename ValueType, typename std::enable_if<
-                 not std::is_pointer<ValueType>::value, int>::type = 0>
-    ValueType get() const
+    template <
+        typename ValueTypeCV,
+        typename ValueType = detail::uncvref_t<ValueTypeCV>,
+        detail::enable_if_t <
+            not std::is_same<basic_json_t, ValueType>::value and
+            detail::has_from_json<basic_json_t, ValueType>::value and
+            not detail::has_non_default_from_json<basic_json_t, ValueType>::value,
+            int > = 0 >
+    ValueType get() const noexcept(noexcept(
+                                       JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), std::declval<ValueType&>())))
     {
-        return get_impl(static_cast<ValueType*>(nullptr));
+        // we cannot static_assert on ValueTypeCV being non-const, because
+        // there is support for get<const basic_json_t>(), which is why we
+        // still need the uncvref
+        static_assert(not std::is_reference<ValueTypeCV>::value,
+                      "get() cannot be used with reference types, you might want to use get_ref()");
+        static_assert(std::is_default_constructible<ValueType>::value,
+                      "types must be DefaultConstructible when used with get()");
+
+        ValueType ret;
+        JSONSerializer<ValueType>::from_json(*this, ret);
+        return ret;
+    }
+
+    /*!
+    @brief get a value (explicit); special case
+
+    Explicit type conversion between the JSON value and a compatible value
+    which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible)
+    and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible).
+    The value is converted by calling the @ref json_serializer<ValueType>
+    `from_json()` method.
+
+    The function is equivalent to executing
+    @code {.cpp}
+    return JSONSerializer<ValueTypeCV>::from_json(*this);
+    @endcode
+
+    This overloads is chosen if:
+    - @a ValueType is not @ref basic_json and
+    - @ref json_serializer<ValueType> has a `from_json()` method of the form
+      `ValueType from_json(const @ref basic_json&)`
+
+    @note If @ref json_serializer<ValueType> has both overloads of
+    `from_json()`, this one is chosen.
+
+    @tparam ValueTypeCV the provided value type
+    @tparam ValueType the returned value type
+
+    @return copy of the JSON value, converted to @a ValueType
+
+    @throw what @ref json_serializer<ValueType> `from_json()` method throws
+
+    @since version 2.1.0
+    */
+    template <
+        typename ValueTypeCV,
+        typename ValueType = detail::uncvref_t<ValueTypeCV>,
+        detail::enable_if_t<not std::is_same<basic_json_t, ValueType>::value and
+                            detail::has_non_default_from_json<basic_json_t,
+                                    ValueType>::value, int> = 0 >
+    ValueType get() const noexcept(noexcept(
+                                       JSONSerializer<ValueTypeCV>::from_json(std::declval<const basic_json_t&>())))
+    {
+        static_assert(not std::is_reference<ValueTypeCV>::value,
+                      "get() cannot be used with reference types, you might want to use get_ref()");
+        return JSONSerializer<ValueTypeCV>::from_json(*this);
     }
 
     /*!
@@ -3026,7 +3410,7 @@ class basic_json
     /*!
     @brief get a reference value (implicit)
 
-    Implict reference access to the internally stored JSON value. No copies
+    Implicit reference access to the internally stored JSON value. No copies
     are made.
 
     @warning Writing data to the referee of the result yields an undefined
@@ -3101,7 +3485,7 @@ class basic_json
     template < typename ValueType, typename std::enable_if <
                    not std::is_pointer<ValueType>::value and
                    not std::is_same<ValueType, typename string_t::value_type>::value
-#ifndef _MSC_VER  // Fix for issue #167 operator<< abiguity under VS2015
+#ifndef _MSC_VER  // fix for issue #167 operator<< ambiguity under VS2015
                    and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value
 #endif
                    , int >::type = 0 >
@@ -3149,19 +3533,19 @@ class basic_json
         // at only works for arrays
         if (is_array())
         {
-            try
+            JSON_TRY
             {
                 return m_value.array->at(idx);
             }
-            catch (std::out_of_range&)
+            JSON_CATCH (std::out_of_range&)
             {
                 // create better exception explanation
-                throw std::out_of_range("array index " + std::to_string(idx) + " is out of range");
+                JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range"));
             }
         }
         else
         {
-            throw std::domain_error("cannot use at() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use at() with " + type_name()));
         }
     }
 
@@ -3192,19 +3576,19 @@ class basic_json
         // at only works for arrays
         if (is_array())
         {
-            try
+            JSON_TRY
             {
                 return m_value.array->at(idx);
             }
-            catch (std::out_of_range&)
+            JSON_CATCH (std::out_of_range&)
             {
                 // create better exception explanation
-                throw std::out_of_range("array index " + std::to_string(idx) + " is out of range");
+                JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range"));
             }
         }
         else
         {
-            throw std::domain_error("cannot use at() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use at() with " + type_name()));
         }
     }
 
@@ -3239,19 +3623,19 @@ class basic_json
         // at only works for objects
         if (is_object())
         {
-            try
+            JSON_TRY
             {
                 return m_value.object->at(key);
             }
-            catch (std::out_of_range&)
+            JSON_CATCH (std::out_of_range&)
             {
                 // create better exception explanation
-                throw std::out_of_range("key '" + key + "' not found");
+                JSON_THROW(std::out_of_range("key '" + key + "' not found"));
             }
         }
         else
         {
-            throw std::domain_error("cannot use at() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use at() with " + type_name()));
         }
     }
 
@@ -3286,19 +3670,19 @@ class basic_json
         // at only works for objects
         if (is_object())
         {
-            try
+            JSON_TRY
             {
                 return m_value.object->at(key);
             }
-            catch (std::out_of_range&)
+            JSON_CATCH (std::out_of_range&)
             {
                 // create better exception explanation
-                throw std::out_of_range("key '" + key + "' not found");
+                JSON_THROW(std::out_of_range("key '" + key + "' not found"));
             }
         }
         else
         {
-            throw std::domain_error("cannot use at() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use at() with " + type_name()));
         }
     }
 
@@ -3350,10 +3734,8 @@ class basic_json
 
             return m_value.array->operator[](idx);
         }
-        else
-        {
-            throw std::domain_error("cannot use operator[] with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3382,10 +3764,8 @@ class basic_json
         {
             return m_value.array->operator[](idx);
         }
-        else
-        {
-            throw std::domain_error("cannot use operator[] with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3430,10 +3810,8 @@ class basic_json
         {
             return m_value.object->operator[](key);
         }
-        else
-        {
-            throw std::domain_error("cannot use operator[] with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3474,10 +3852,8 @@ class basic_json
             assert(m_value.object->find(key) != m_value.object->end());
             return m_value.object->find(key)->second;
         }
-        else
-        {
-            throw std::domain_error("cannot use operator[] with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3591,10 +3967,8 @@ class basic_json
         {
             return m_value.object->operator[](key);
         }
-        else
-        {
-            throw std::domain_error("cannot use operator[] with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3636,10 +4010,8 @@ class basic_json
             assert(m_value.object->find(key) != m_value.object->end());
             return m_value.object->find(key)->second;
         }
-        else
-        {
-            throw std::domain_error("cannot use operator[] with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3703,14 +4075,12 @@ class basic_json
             {
                 return *it;
             }
-            else
-            {
-                return default_value;
-            }
+
+            return default_value;
         }
         else
         {
-            throw std::domain_error("cannot use value() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use value() with " + type_name()));
         }
     }
 
@@ -3772,19 +4142,17 @@ class basic_json
         if (is_object())
         {
             // if pointer resolves a value, return it or use default value
-            try
+            JSON_TRY
             {
                 return ptr.get_checked(this);
             }
-            catch (std::out_of_range&)
+            JSON_CATCH (std::out_of_range&)
             {
                 return default_value;
             }
         }
-        else
-        {
-            throw std::domain_error("cannot use value() with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use value() with " + type_name()));
     }
 
     /*!
@@ -3910,7 +4278,7 @@ class basic_json
 
     @complexity The complexity depends on the type:
     - objects: amortized constant
-    - arrays: linear in distance between pos and the end of the container
+    - arrays: linear in distance between @a pos and the end of the container
     - strings: linear in the length of the string
     - other types: constant
 
@@ -3935,7 +4303,7 @@ class basic_json
         // make sure iterator fits the current value
         if (this != pos.m_object)
         {
-            throw std::domain_error("iterator does not fit current value");
+            JSON_THROW(std::domain_error("iterator does not fit current value"));
         }
 
         IteratorType result = end();
@@ -3950,7 +4318,7 @@ class basic_json
             {
                 if (not pos.m_it.primitive_iterator.is_begin())
                 {
-                    throw std::out_of_range("iterator out of range");
+                    JSON_THROW(std::out_of_range("iterator out of range"));
                 }
 
                 if (is_string())
@@ -3980,7 +4348,7 @@ class basic_json
 
             default:
             {
-                throw std::domain_error("cannot use erase() with " + type_name());
+                JSON_THROW(std::domain_error("cannot use erase() with " + type_name()));
             }
         }
 
@@ -4042,7 +4410,7 @@ class basic_json
         // make sure iterator fits the current value
         if (this != first.m_object or this != last.m_object)
         {
-            throw std::domain_error("iterators do not fit current value");
+            JSON_THROW(std::domain_error("iterators do not fit current value"));
         }
 
         IteratorType result = end();
@@ -4057,7 +4425,7 @@ class basic_json
             {
                 if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end())
                 {
-                    throw std::out_of_range("iterators out of range");
+                    JSON_THROW(std::out_of_range("iterators out of range"));
                 }
 
                 if (is_string())
@@ -4089,7 +4457,7 @@ class basic_json
 
             default:
             {
-                throw std::domain_error("cannot use erase() with " + type_name());
+                JSON_THROW(std::domain_error("cannot use erase() with " + type_name()));
             }
         }
 
@@ -4132,10 +4500,8 @@ class basic_json
         {
             return m_value.object->erase(key);
         }
-        else
-        {
-            throw std::domain_error("cannot use erase() with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use erase() with " + type_name()));
     }
 
     /*!
@@ -4169,14 +4535,14 @@ class basic_json
         {
             if (idx >= size())
             {
-                throw std::out_of_range("array index " + std::to_string(idx) + " is out of range");
+                JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range"));
             }
 
             m_value.array->erase(m_value.array->begin() + static_cast<difference_type>(idx));
         }
         else
         {
-            throw std::domain_error("cannot use erase() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use erase() with " + type_name()));
         }
     }
 
@@ -4894,7 +5260,7 @@ class basic_json
         // push_back only works for null objects or arrays
         if (not(is_null() or is_array()))
         {
-            throw std::domain_error("cannot use push_back() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use push_back() with " + type_name()));
         }
 
         // transform null object into an array
@@ -4930,7 +5296,7 @@ class basic_json
         // push_back only works for null objects or arrays
         if (not(is_null() or is_array()))
         {
-            throw std::domain_error("cannot use push_back() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use push_back() with " + type_name()));
         }
 
         // transform null object into an array
@@ -4980,7 +5346,7 @@ class basic_json
         // push_back only works for null objects or objects
         if (not(is_null() or is_object()))
         {
-            throw std::domain_error("cannot use push_back() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use push_back() with " + type_name()));
         }
 
         // transform null object into an object
@@ -5080,7 +5446,7 @@ class basic_json
         // emplace_back only works for null objects or arrays
         if (not(is_null() or is_array()))
         {
-            throw std::domain_error("cannot use emplace_back() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use emplace_back() with " + type_name()));
         }
 
         // transform null object into an array
@@ -5098,8 +5464,8 @@ class basic_json
     /*!
     @brief add an object to an object if key does not exist
 
-    Inserts a new element into a JSON object constructed in-place with the given
-    @a args if there is no element with the key in the container. If the
+    Inserts a new element into a JSON object constructed in-place with the
+    given @a args if there is no element with the key in the container. If the
     function is called on a JSON null value, an empty object is created before
     appending the value created from @a args.
 
@@ -5128,7 +5494,7 @@ class basic_json
         // emplace only works for null objects or arrays
         if (not(is_null() or is_object()))
         {
-            throw std::domain_error("cannot use emplace() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use emplace() with " + type_name()));
         }
 
         // transform null object into an object
@@ -5164,8 +5530,8 @@ class basic_json
     @throw std::domain_error if @a pos is not an iterator of *this; example:
     `"iterator does not fit current value"`
 
-    @complexity Constant plus linear in the distance between pos and end of the
-    container.
+    @complexity Constant plus linear in the distance between @a pos and end of
+    the container.
 
     @liveexample{The example shows how `insert()` is used.,insert}
 
@@ -5179,7 +5545,7 @@ class basic_json
             // check if iterator pos fits to this JSON value
             if (pos.m_object != this)
             {
-                throw std::domain_error("iterator does not fit current value");
+                JSON_THROW(std::domain_error("iterator does not fit current value"));
             }
 
             // insert to array and return iterator
@@ -5187,10 +5553,8 @@ class basic_json
             result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val);
             return result;
         }
-        else
-        {
-            throw std::domain_error("cannot use insert() with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use insert() with " + type_name()));
     }
 
     /*!
@@ -5234,7 +5598,7 @@ class basic_json
             // check if iterator pos fits to this JSON value
             if (pos.m_object != this)
             {
-                throw std::domain_error("iterator does not fit current value");
+                JSON_THROW(std::domain_error("iterator does not fit current value"));
             }
 
             // insert to array and return iterator
@@ -5242,10 +5606,8 @@ class basic_json
             result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val);
             return result;
         }
-        else
-        {
-            throw std::domain_error("cannot use insert() with " + type_name());
-        }
+
+        JSON_THROW(std::domain_error("cannot use insert() with " + type_name()));
     }
 
     /*!
@@ -5283,24 +5645,24 @@ class basic_json
         // insert only works for arrays
         if (not is_array())
         {
-            throw std::domain_error("cannot use insert() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use insert() with " + type_name()));
         }
 
         // check if iterator pos fits to this JSON value
         if (pos.m_object != this)
         {
-            throw std::domain_error("iterator does not fit current value");
+            JSON_THROW(std::domain_error("iterator does not fit current value"));
         }
 
         // check if range iterators belong to the same JSON object
         if (first.m_object != last.m_object)
         {
-            throw std::domain_error("iterators do not fit");
+            JSON_THROW(std::domain_error("iterators do not fit"));
         }
 
         if (first.m_object == this or last.m_object == this)
         {
-            throw std::domain_error("passed iterators may not belong to container");
+            JSON_THROW(std::domain_error("passed iterators may not belong to container"));
         }
 
         // insert to array and return iterator
@@ -5341,13 +5703,13 @@ class basic_json
         // insert only works for arrays
         if (not is_array())
         {
-            throw std::domain_error("cannot use insert() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use insert() with " + type_name()));
         }
 
         // check if iterator pos fits to this JSON value
         if (pos.m_object != this)
         {
-            throw std::domain_error("iterator does not fit current value");
+            JSON_THROW(std::domain_error("iterator does not fit current value"));
         }
 
         // insert to array and return iterator
@@ -5395,8 +5757,8 @@ class basic_json
 
     @param[in,out] other array to exchange the contents with
 
-    @throw std::domain_error when JSON value is not an array; example: `"cannot
-    use swap() with string"`
+    @throw std::domain_error when JSON value is not an array; example:
+    `"cannot use swap() with string"`
 
     @complexity Constant.
 
@@ -5414,7 +5776,7 @@ class basic_json
         }
         else
         {
-            throw std::domain_error("cannot use swap() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use swap() with " + type_name()));
         }
     }
 
@@ -5447,7 +5809,7 @@ class basic_json
         }
         else
         {
-            throw std::domain_error("cannot use swap() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use swap() with " + type_name()));
         }
     }
 
@@ -5480,13 +5842,13 @@ class basic_json
         }
         else
         {
-            throw std::domain_error("cannot use swap() with " + type_name());
+            JSON_THROW(std::domain_error("cannot use swap() with " + type_name()));
         }
     }
 
     /// @}
 
-
+  public:
     //////////////////////////////////////////
     // lexicographical comparison operators //
     //////////////////////////////////////////
@@ -5494,40 +5856,6 @@ class basic_json
     /// @name lexicographical comparison operators
     /// @{
 
-  private:
-    /*!
-    @brief comparison operator for JSON types
-
-    Returns an ordering that is similar to Python:
-    - order: null < boolean < number < object < array < string
-    - furthermore, each type is not smaller than itself
-
-    @since version 1.0.0
-    */
-    friend bool operator<(const value_t lhs, const value_t rhs) noexcept
-    {
-        static constexpr std::array<uint8_t, 8> order = {{
-                0, // null
-                3, // object
-                4, // array
-                5, // string
-                1, // boolean
-                2, // integer
-                2, // unsigned
-                2, // float
-            }
-        };
-
-        // discarded values are not comparable
-        if (lhs == value_t::discarded or rhs == value_t::discarded)
-        {
-            return false;
-        }
-
-        return order[static_cast<std::size_t>(lhs)] < order[static_cast<std::size_t>(rhs)];
-    }
-
-  public:
     /*!
     @brief comparison: equal
 
@@ -5628,34 +5956,24 @@ class basic_json
 
     /*!
     @brief comparison: equal
-
-    The functions compares the given JSON value against a null pointer. As the
-    null pointer can be used to initialize a JSON value to null, a comparison
-    of JSON value @a v with a null pointer should be equivalent to call
-    `v.is_null()`.
-
-    @param[in] v  JSON value to consider
-    @return whether @a v is null
-
-    @complexity Constant.
-
-    @liveexample{The example compares several JSON types to the null pointer.
-    ,operator__equal__nullptr_t}
-
-    @since version 1.0.0
+    @copydoc operator==(const_reference, const_reference)
     */
-    friend bool operator==(const_reference v, std::nullptr_t) noexcept
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept
     {
-        return v.is_null();
+        return (lhs == basic_json(rhs));
     }
 
     /*!
     @brief comparison: equal
-    @copydoc operator==(const_reference, std::nullptr_t)
+    @copydoc operator==(const_reference, const_reference)
     */
-    friend bool operator==(std::nullptr_t, const_reference v) noexcept
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept
     {
-        return v.is_null();
+        return (basic_json(lhs) == rhs);
     }
 
     /*!
@@ -5681,34 +5999,24 @@ class basic_json
 
     /*!
     @brief comparison: not equal
-
-    The functions compares the given JSON value against a null pointer. As the
-    null pointer can be used to initialize a JSON value to null, a comparison
-    of JSON value @a v with a null pointer should be equivalent to call
-    `not v.is_null()`.
-
-    @param[in] v  JSON value to consider
-    @return whether @a v is not null
-
-    @complexity Constant.
-
-    @liveexample{The example compares several JSON types to the null pointer.
-    ,operator__notequal__nullptr_t}
-
-    @since version 1.0.0
+    @copydoc operator!=(const_reference, const_reference)
     */
-    friend bool operator!=(const_reference v, std::nullptr_t) noexcept
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept
     {
-        return not v.is_null();
+        return (lhs != basic_json(rhs));
     }
 
     /*!
     @brief comparison: not equal
-    @copydoc operator!=(const_reference, std::nullptr_t)
+    @copydoc operator!=(const_reference, const_reference)
     */
-    friend bool operator!=(std::nullptr_t, const_reference v) noexcept
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept
     {
-        return not v.is_null();
+        return (basic_json(lhs) != rhs);
     }
 
     /*!
@@ -5899,10 +6207,6 @@ class basic_json
     `std::setw(4)` on @a o sets the indentation level to `4` and the
     serialization result is the same as calling `dump(4)`.
 
-    @note During serializaion, the locale and the precision of the output
-    stream @a o are changed. The original values are restored when the
-    function returns.
-
     @param[in,out] o  stream to serialize to
     @param[in] j  JSON value to serialize
 
@@ -5924,22 +6228,9 @@ class basic_json
         // reset width to 0 for subsequent calls to this stream
         o.width(0);
 
-        // fix locale problems
-        const auto old_locale = o.imbue(std::locale::classic());
-        // set precision
-
-        // 6, 15 or 16 digits of precision allows round-trip IEEE 754
-        // string->float->string, string->double->string or string->long
-        // double->string; to be safe, we read this value from
-        // std::numeric_limits<number_float_t>::digits10
-        const auto old_precision = o.precision(std::numeric_limits<double>::digits10);
-
         // do the actual serialization
         j.dump(o, pretty_print, static_cast<unsigned int>(indentation));
 
-        // reset locale and precision
-        o.imbue(old_locale);
-        o.precision(old_precision);
         return o;
     }
 
@@ -6123,7 +6414,7 @@ class basic_json
     {
         // assertion to check that the iterator range is indeed contiguous,
         // see http://stackoverflow.com/a/35008842/266378 for more discussion
-        assert(std::accumulate(first, last, std::make_pair<bool, int>(true, 0),
+        assert(std::accumulate(first, last, std::pair<bool, int>(true, 0),
                                [&first](std::pair<bool, int> res, decltype(*first) val)
         {
             res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
@@ -6246,6 +6537,11 @@ class basic_json
     /// @{
 
   private:
+    /*!
+    @note Some code in the switch cases has been copied, because otherwise
+          copilers would complain about implicit fallthrough and there is no
+          portable attribute to mute such warnings.
+    */
     template<typename T>
     static void add_to_vector(std::vector<uint8_t>& vec, size_t bytes, const T number)
     {
@@ -6255,24 +6551,31 @@ class basic_json
         {
             case 8:
             {
-                vec.push_back(static_cast<uint8_t>((number >> 070) & 0xff));
-                vec.push_back(static_cast<uint8_t>((number >> 060) & 0xff));
-                vec.push_back(static_cast<uint8_t>((number >> 050) & 0xff));
-                vec.push_back(static_cast<uint8_t>((number >> 040) & 0xff));
-                // intentional fall-through
+                vec.push_back(static_cast<uint8_t>((static_cast<uint64_t>(number) >> 070) & 0xff));
+                vec.push_back(static_cast<uint8_t>((static_cast<uint64_t>(number) >> 060) & 0xff));
+                vec.push_back(static_cast<uint8_t>((static_cast<uint64_t>(number) >> 050) & 0xff));
+                vec.push_back(static_cast<uint8_t>((static_cast<uint64_t>(number) >> 040) & 0xff));
+                vec.push_back(static_cast<uint8_t>((number >> 030) & 0xff));
+                vec.push_back(static_cast<uint8_t>((number >> 020) & 0xff));
+                vec.push_back(static_cast<uint8_t>((number >> 010) & 0xff));
+                vec.push_back(static_cast<uint8_t>(number & 0xff));
+                break;
             }
 
             case 4:
             {
                 vec.push_back(static_cast<uint8_t>((number >> 030) & 0xff));
                 vec.push_back(static_cast<uint8_t>((number >> 020) & 0xff));
-                // intentional fall-through
+                vec.push_back(static_cast<uint8_t>((number >> 010) & 0xff));
+                vec.push_back(static_cast<uint8_t>(number & 0xff));
+                break;
             }
 
             case 2:
             {
                 vec.push_back(static_cast<uint8_t>((number >> 010) & 0xff));
-                // intentional fall-through
+                vec.push_back(static_cast<uint8_t>(number & 0xff));
+                break;
             }
 
             case 1:
@@ -6324,11 +6627,11 @@ class basic_json
     {
         if (current_index + sizeof(T) + 1 > vec.size())
         {
-            throw std::out_of_range("cannot read " + std::to_string(sizeof(T)) + " bytes from vector");
+            JSON_THROW(std::out_of_range("cannot read " + std::to_string(sizeof(T)) + " bytes from vector"));
         }
 
         T result;
-        uint8_t* ptr = reinterpret_cast<uint8_t*>(&result);
+        auto* ptr = reinterpret_cast<uint8_t*>(&result);
         for (size_t i = 0; i < sizeof(T); ++i)
         {
             *ptr++ = vec[current_index + sizeof(T) - i];
@@ -6369,32 +6672,33 @@ class basic_json
                 if (j.m_value.number_integer >= 0)
                 {
                     // MessagePack does not differentiate between positive
-                    // signed integers and unsigned integers. Therefore, we used
-                    // the code from the value_t::number_unsigned case here.
+                    // signed integers and unsigned integers. Therefore, we
+                    // used the code from the value_t::number_unsigned case
+                    // here.
                     if (j.m_value.number_unsigned < 128)
                     {
                         // positive fixnum
                         add_to_vector(v, 1, j.m_value.number_unsigned);
                     }
-                    else if (j.m_value.number_unsigned <= UINT8_MAX)
+                    else if (j.m_value.number_unsigned <= std::numeric_limits<uint8_t>::max())
                     {
                         // uint 8
                         v.push_back(0xcc);
                         add_to_vector(v, 1, j.m_value.number_unsigned);
                     }
-                    else if (j.m_value.number_unsigned <= UINT16_MAX)
+                    else if (j.m_value.number_unsigned <= std::numeric_limits<uint16_t>::max())
                     {
                         // uint 16
                         v.push_back(0xcd);
                         add_to_vector(v, 2, j.m_value.number_unsigned);
                     }
-                    else if (j.m_value.number_unsigned <= UINT32_MAX)
+                    else if (j.m_value.number_unsigned <= std::numeric_limits<uint32_t>::max())
                     {
                         // uint 32
                         v.push_back(0xce);
                         add_to_vector(v, 4, j.m_value.number_unsigned);
                     }
-                    else if (j.m_value.number_unsigned <= UINT64_MAX)
+                    else if (j.m_value.number_unsigned <= std::numeric_limits<uint64_t>::max())
                     {
                         // uint 64
                         v.push_back(0xcf);
@@ -6408,25 +6712,25 @@ class basic_json
                         // negative fixnum
                         add_to_vector(v, 1, j.m_value.number_integer);
                     }
-                    else if (j.m_value.number_integer >= INT8_MIN and j.m_value.number_integer <= INT8_MAX)
+                    else if (j.m_value.number_integer >= std::numeric_limits<int8_t>::min() and j.m_value.number_integer <= std::numeric_limits<int8_t>::max())
                     {
                         // int 8
                         v.push_back(0xd0);
                         add_to_vector(v, 1, j.m_value.number_integer);
                     }
-                    else if (j.m_value.number_integer >= INT16_MIN and j.m_value.number_integer <= INT16_MAX)
+                    else if (j.m_value.number_integer >= std::numeric_limits<int16_t>::min() and j.m_value.number_integer <= std::numeric_limits<int16_t>::max())
                     {
                         // int 16
                         v.push_back(0xd1);
                         add_to_vector(v, 2, j.m_value.number_integer);
                     }
-                    else if (j.m_value.number_integer >= INT32_MIN and j.m_value.number_integer <= INT32_MAX)
+                    else if (j.m_value.number_integer >= std::numeric_limits<int32_t>::min() and j.m_value.number_integer <= std::numeric_limits<int32_t>::max())
                     {
                         // int 32
                         v.push_back(0xd2);
                         add_to_vector(v, 4, j.m_value.number_integer);
                     }
-                    else if (j.m_value.number_integer >= INT64_MIN and j.m_value.number_integer <= INT64_MAX)
+                    else if (j.m_value.number_integer >= std::numeric_limits<int64_t>::min() and j.m_value.number_integer <= std::numeric_limits<int64_t>::max())
                     {
                         // int 64
                         v.push_back(0xd3);
@@ -6443,25 +6747,25 @@ class basic_json
                     // positive fixnum
                     add_to_vector(v, 1, j.m_value.number_unsigned);
                 }
-                else if (j.m_value.number_unsigned <= UINT8_MAX)
+                else if (j.m_value.number_unsigned <= std::numeric_limits<uint8_t>::max())
                 {
                     // uint 8
                     v.push_back(0xcc);
                     add_to_vector(v, 1, j.m_value.number_unsigned);
                 }
-                else if (j.m_value.number_unsigned <= UINT16_MAX)
+                else if (j.m_value.number_unsigned <= std::numeric_limits<uint16_t>::max())
                 {
                     // uint 16
                     v.push_back(0xcd);
                     add_to_vector(v, 2, j.m_value.number_unsigned);
                 }
-                else if (j.m_value.number_unsigned <= UINT32_MAX)
+                else if (j.m_value.number_unsigned <= std::numeric_limits<uint32_t>::max())
                 {
                     // uint 32
                     v.push_back(0xce);
                     add_to_vector(v, 4, j.m_value.number_unsigned);
                 }
-                else if (j.m_value.number_unsigned <= UINT64_MAX)
+                else if (j.m_value.number_unsigned <= std::numeric_limits<uint64_t>::max())
                 {
                     // uint 64
                     v.push_back(0xcf);
@@ -6474,7 +6778,7 @@ class basic_json
             {
                 // float 64
                 v.push_back(0xcb);
-                const uint8_t* helper = reinterpret_cast<const uint8_t*>(&(j.m_value.number_float));
+                const auto* helper = reinterpret_cast<const uint8_t*>(&(j.m_value.number_float));
                 for (size_t i = 0; i < 8; ++i)
                 {
                     v.push_back(helper[7 - i]);
@@ -6618,19 +6922,19 @@ class basic_json
                     {
                         add_to_vector(v, 1, j.m_value.number_integer);
                     }
-                    else if (j.m_value.number_integer <= UINT8_MAX)
+                    else if (j.m_value.number_integer <= std::numeric_limits<uint8_t>::max())
                     {
                         v.push_back(0x18);
                         // one-byte uint8_t
                         add_to_vector(v, 1, j.m_value.number_integer);
                     }
-                    else if (j.m_value.number_integer <= UINT16_MAX)
+                    else if (j.m_value.number_integer <= std::numeric_limits<uint16_t>::max())
                     {
                         v.push_back(0x19);
                         // two-byte uint16_t
                         add_to_vector(v, 2, j.m_value.number_integer);
                     }
-                    else if (j.m_value.number_integer <= UINT32_MAX)
+                    else if (j.m_value.number_integer <= std::numeric_limits<uint32_t>::max())
                     {
                         v.push_back(0x1a);
                         // four-byte uint32_t
@@ -6645,26 +6949,26 @@ class basic_json
                 }
                 else
                 {
-                    // The conversions below encode the sign in the first byte,
-                    // and the value is converted to a positive number.
+                    // The conversions below encode the sign in the first
+                    // byte, and the value is converted to a positive number.
                     const auto positive_number = -1 - j.m_value.number_integer;
                     if (j.m_value.number_integer >= -24)
                     {
                         v.push_back(static_cast<uint8_t>(0x20 + positive_number));
                     }
-                    else if (positive_number <= UINT8_MAX)
+                    else if (positive_number <= std::numeric_limits<uint8_t>::max())
                     {
                         // int 8
                         v.push_back(0x38);
                         add_to_vector(v, 1, positive_number);
                     }
-                    else if (positive_number <= UINT16_MAX)
+                    else if (positive_number <= std::numeric_limits<uint16_t>::max())
                     {
                         // int 16
                         v.push_back(0x39);
                         add_to_vector(v, 2, positive_number);
                     }
-                    else if (positive_number <= UINT32_MAX)
+                    else if (positive_number <= std::numeric_limits<uint32_t>::max())
                     {
                         // int 32
                         v.push_back(0x3a);
@@ -6717,7 +7021,7 @@ class basic_json
             {
                 // Double-Precision Float
                 v.push_back(0xfb);
-                const uint8_t* helper = reinterpret_cast<const uint8_t*>(&(j.m_value.number_float));
+                const auto* helper = reinterpret_cast<const uint8_t*>(&(j.m_value.number_float));
                 for (size_t i = 0; i < 8; ++i)
                 {
                     v.push_back(helper[7 - i]);
@@ -6730,7 +7034,7 @@ class basic_json
                 const auto N = j.m_value.string->size();
                 if (N <= 0x17)
                 {
-                    v.push_back(0x60 + N);  // 1 byte for string + size
+                    v.push_back(0x60 + static_cast<uint8_t>(N));  // 1 byte for string + size
                 }
                 else if (N <= 0xff)
                 {
@@ -6766,7 +7070,7 @@ class basic_json
                 const auto N = j.m_value.array->size();
                 if (N <= 0x17)
                 {
-                    v.push_back(0x80 + N);  // 1 byte for array + size
+                    v.push_back(0x80 + static_cast<uint8_t>(N));  // 1 byte for array + size
                 }
                 else if (N <= 0xff)
                 {
@@ -6804,7 +7108,7 @@ class basic_json
                 const auto N = j.m_value.object->size();
                 if (N <= 0x17)
                 {
-                    v.push_back(0xa0 + N);  // 1 byte for object + size
+                    v.push_back(0xa0 + static_cast<uint8_t>(N));  // 1 byte for object + size
                 }
                 else if (N <= 0xff)
                 {
@@ -6851,12 +7155,12 @@ class basic_json
 
     To secure the access to the byte vector during CBOR/MessagePack
     deserialization, bytes are copied from the vector into buffers. This
-    function checks if the number of bytes to copy (@a len) does not exceed the
-    size @s size of the vector. Additionally, an @a offset is given from where
-    to start reading the bytes.
+    function checks if the number of bytes to copy (@a len) does not exceed
+    the size @s size of the vector. Additionally, an @a offset is given from
+    where to start reading the bytes.
 
-    This function checks whether reading the bytes is safe; that is, offset is a
-    valid index in the vector, offset+len
+    This function checks whether reading the bytes is safe; that is, offset is
+    a valid index in the vector, offset+len
 
     @param[in] size    size of the byte vector
     @param[in] len     number of bytes to read
@@ -6873,19 +7177,19 @@ class basic_json
         // simple case: requested length is greater than the vector's length
         if (len > size or offset > size)
         {
-            throw std::out_of_range("len out of range");
+            JSON_THROW(std::out_of_range("len out of range"));
         }
 
         // second case: adding offset would result in overflow
         if ((size > (std::numeric_limits<size_t>::max() - offset)))
         {
-            throw std::out_of_range("len+offset out of range");
+            JSON_THROW(std::out_of_range("len+offset out of range"));
         }
 
         // last case: reading past the end of the vector
         if (len + offset > size)
         {
-            throw std::out_of_range("len+offset out of range");
+            JSON_THROW(std::out_of_range("len+offset out of range"));
         }
     }
 
@@ -6917,7 +7221,7 @@ class basic_json
             {
                 return v[current_idx];
             }
-            else if (v[current_idx] <= 0x8f) // fixmap
+            if (v[current_idx] <= 0x8f) // fixmap
             {
                 basic_json result = value_t::object;
                 const size_t len = v[current_idx] & 0x0f;
@@ -6973,11 +7277,10 @@ class basic_json
                 case 0xca: // float 32
                 {
                     // copy bytes in reverse order into the double variable
-                    check_length(v.size(), sizeof(float), 1);
                     float res;
                     for (size_t byte = 0; byte < sizeof(float); ++byte)
                     {
-                        reinterpret_cast<uint8_t*>(&res)[sizeof(float) - byte - 1] = v[current_idx + 1 + byte];
+                        reinterpret_cast<uint8_t*>(&res)[sizeof(float) - byte - 1] = v.at(current_idx + 1 + byte);
                     }
                     idx += sizeof(float); // skip content bytes
                     return res;
@@ -6986,11 +7289,10 @@ class basic_json
                 case 0xcb: // float 64
                 {
                     // copy bytes in reverse order into the double variable
-                    check_length(v.size(), sizeof(double), 1);
                     double res;
                     for (size_t byte = 0; byte < sizeof(double); ++byte)
                     {
-                        reinterpret_cast<uint8_t*>(&res)[sizeof(double) - byte - 1] = v[current_idx + 1 + byte];
+                        reinterpret_cast<uint8_t*>(&res)[sizeof(double) - byte - 1] = v.at(current_idx + 1 + byte);
                     }
                     idx += sizeof(double); // skip content bytes
                     return res;
@@ -7123,7 +7425,7 @@ class basic_json
 
                 default:
                 {
-                    throw std::invalid_argument("error parsing a msgpack @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast<int>(v[current_idx])));
+                    JSON_THROW(std::invalid_argument("error parsing a msgpack @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast<int>(v[current_idx]))));
                 }
             }
         }
@@ -7552,7 +7854,6 @@ class basic_json
 
             case 0xf9: // Half-Precision Float (two-byte IEEE 754)
             {
-                check_length(v.size(), 2, 1);
                 idx += 2; // skip two content bytes
 
                 // code from RFC 7049, Appendix D, Figure 3:
@@ -7562,7 +7863,7 @@ class basic_json
                 // include at least decoding support for them even without such
                 // support. An example of a small decoder for half-precision
                 // floating-point numbers in the C language is shown in Fig. 3.
-                const int half = (v[current_idx + 1] << 8) + v[current_idx + 2];
+                const int half = (v.at(current_idx + 1) << 8) + v.at(current_idx + 2);
                 const int exp = (half >> 10) & 0x1f;
                 const int mant = half & 0x3ff;
                 double val;
@@ -7576,19 +7877,20 @@ class basic_json
                 }
                 else
                 {
-                    val = mant == 0 ? INFINITY : NAN;
+                    val = mant == 0
+                          ? std::numeric_limits<double>::infinity()
+                          : std::numeric_limits<double>::quiet_NaN();
                 }
-                return half & 0x8000 ? -val : val;
+                return (half & 0x8000) != 0 ? -val : val;
             }
 
             case 0xfa: // Single-Precision Float (four-byte IEEE 754)
             {
                 // copy bytes in reverse order into the float variable
-                check_length(v.size(), sizeof(float), 1);
                 float res;
                 for (size_t byte = 0; byte < sizeof(float); ++byte)
                 {
-                    reinterpret_cast<uint8_t*>(&res)[sizeof(float) - byte - 1] = v[current_idx + 1 + byte];
+                    reinterpret_cast<uint8_t*>(&res)[sizeof(float) - byte - 1] = v.at(current_idx + 1 + byte);
                 }
                 idx += sizeof(float); // skip content bytes
                 return res;
@@ -7596,12 +7898,11 @@ class basic_json
 
             case 0xfb: // Double-Precision Float (eight-byte IEEE 754)
             {
-                check_length(v.size(), sizeof(double), 1);
                 // copy bytes in reverse order into the double variable
                 double res;
                 for (size_t byte = 0; byte < sizeof(double); ++byte)
                 {
-                    reinterpret_cast<uint8_t*>(&res)[sizeof(double) - byte - 1] = v[current_idx + 1 + byte];
+                    reinterpret_cast<uint8_t*>(&res)[sizeof(double) - byte - 1] = v.at(current_idx + 1 + byte);
                 }
                 idx += sizeof(double); // skip content bytes
                 return res;
@@ -7609,7 +7910,7 @@ class basic_json
 
             default: // anything else (0xFF is handled inside the other types)
             {
-                throw std::invalid_argument("error parsing a CBOR @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast<int>(v[current_idx])));
+                JSON_THROW(std::invalid_argument("error parsing a CBOR @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast<int>(v[current_idx]))));
             }
         }
     }
@@ -7631,9 +7932,11 @@ class basic_json
     vector in MessagePack format.,to_msgpack}
 
     @sa http://msgpack.org
-    @sa @ref from_msgpack(const std::vector<uint8_t>&) for the analogous
-        deserialization
+    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
+        analogous deserialization
     @sa @ref to_cbor(const basic_json& for the related CBOR format
+
+    @since version 2.0.9
     */
     static std::vector<uint8_t> to_msgpack(const basic_json& j)
     {
@@ -7649,6 +7952,7 @@ class basic_json
     serialization format.
 
     @param[in] v  a byte vector in MessagePack format
+    @param[in] start_index the index to start reading from @a v (0 by default)
     @return deserialized JSON value
 
     @throw std::invalid_argument if unsupported features from MessagePack were
@@ -7662,11 +7966,15 @@ class basic_json
 
     @sa http://msgpack.org
     @sa @ref to_msgpack(const basic_json&) for the analogous serialization
-    @sa @ref from_cbor(const std::vector<uint8_t>&) for the related CBOR format
+    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
+        related CBOR format
+
+    @since version 2.0.9, parameter @a start_index since 2.1.1
     */
-    static basic_json from_msgpack(const std::vector<uint8_t>& v)
+    static basic_json from_msgpack(const std::vector<uint8_t>& v,
+                                   const size_t start_index = 0)
     {
-        size_t i = 0;
+        size_t i = start_index;
         return from_msgpack_internal(v, i);
     }
 
@@ -7687,9 +7995,11 @@ class basic_json
     vector in CBOR format.,to_cbor}
 
     @sa http://cbor.io
-    @sa @ref from_cbor(const std::vector<uint8_t>&) for the analogous
-        deserialization
+    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
+        analogous deserialization
     @sa @ref to_msgpack(const basic_json& for the related MessagePack format
+
+    @since version 2.0.9
     */
     static std::vector<uint8_t> to_cbor(const basic_json& j)
     {
@@ -7705,6 +8015,7 @@ class basic_json
     (Concise Binary Object Representation) serialization format.
 
     @param[in] v  a byte vector in CBOR format
+    @param[in] start_index the index to start reading from @a v (0 by default)
     @return deserialized JSON value
 
     @throw std::invalid_argument if unsupported features from CBOR were used in
@@ -7718,18 +8029,20 @@ class basic_json
 
     @sa http://cbor.io
     @sa @ref to_cbor(const basic_json&) for the analogous serialization
-    @sa @ref from_msgpack(const std::vector<uint8_t>&) for the related
-        MessagePack format
+    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
+        related MessagePack format
+
+    @since version 2.0.9, parameter @a start_index since 2.1.1
     */
-    static basic_json from_cbor(const std::vector<uint8_t>& v)
+    static basic_json from_cbor(const std::vector<uint8_t>& v,
+                                const size_t start_index = 0)
     {
-        size_t i = 0;
+        size_t i = start_index;
         return from_cbor_internal(v, i);
     }
 
     /// @}
 
-  private:
     ///////////////////////////
     // convenience functions //
     ///////////////////////////
@@ -7744,29 +8057,35 @@ class basic_json
 
     @complexity Constant.
 
-    @since version 1.0.0
+    @liveexample{The following code exemplifies `type_name()` for all JSON
+    types.,type_name}
+
+    @since version 1.0.0, public since 2.1.0
     */
     std::string type_name() const
     {
-        switch (m_type)
         {
-            case value_t::null:
-                return "null";
-            case value_t::object:
-                return "object";
-            case value_t::array:
-                return "array";
-            case value_t::string:
-                return "string";
-            case value_t::boolean:
-                return "boolean";
-            case value_t::discarded:
-                return "discarded";
-            default:
-                return "number";
+            switch (m_type)
+            {
+                case value_t::null:
+                    return "null";
+                case value_t::object:
+                    return "object";
+                case value_t::array:
+                    return "array";
+                case value_t::string:
+                    return "string";
+                case value_t::boolean:
+                    return "boolean";
+                case value_t::discarded:
+                    return "discarded";
+                default:
+                    return "number";
+            }
         }
     }
 
+  private:
     /*!
     @brief calculates the extra space to escape a JSON string
 
@@ -7801,10 +8120,8 @@ class basic_json
                         // from c (1 byte) to \uxxxx (6 bytes)
                         return res + 5;
                     }
-                    else
-                    {
-                        return res;
-                    }
+
+                    return res;
                 }
             }
         });
@@ -7930,6 +8247,154 @@ class basic_json
         return result;
     }
 
+
+    /*!
+    @brief locale-independent serialization for built-in arithmetic types
+    */
+    struct numtostr
+    {
+      public:
+        template<typename NumberType>
+        numtostr(NumberType value)
+        {
+            x_write(value, std::is_integral<NumberType>());
+        }
+
+        const char* c_str() const
+        {
+            return m_buf.data();
+        }
+
+      private:
+        /// a (hopefully) large enough character buffer
+        std::array < char, 64 > m_buf{{}};
+
+        template<typename NumberType>
+        void x_write(NumberType x, /*is_integral=*/std::true_type)
+        {
+            // special case for "0"
+            if (x == 0)
+            {
+                m_buf[0] = '0';
+                return;
+            }
+
+            const bool is_negative = x < 0;
+            size_t i = 0;
+
+            // spare 1 byte for '\0'
+            while (x != 0 and i < m_buf.size() - 1)
+            {
+                const auto digit = std::labs(static_cast<long>(x % 10));
+                m_buf[i++] = static_cast<char>('0' + digit);
+                x /= 10;
+            }
+
+            // make sure the number has been processed completely
+            assert(x == 0);
+
+            if (is_negative)
+            {
+                // make sure there is capacity for the '-'
+                assert(i < m_buf.size() - 2);
+                m_buf[i++] = '-';
+            }
+
+            std::reverse(m_buf.begin(), m_buf.begin() + i);
+        }
+
+        template<typename NumberType>
+        void x_write(NumberType x, /*is_integral=*/std::false_type)
+        {
+            // special case for 0.0 and -0.0
+            if (x == 0)
+            {
+                size_t i = 0;
+                if (std::signbit(x))
+                {
+                    m_buf[i++] = '-';
+                }
+                m_buf[i++] = '0';
+                m_buf[i++] = '.';
+                m_buf[i] = '0';
+                return;
+            }
+
+            // get number of digits for a text -> float -> text round-trip
+            static constexpr auto d = std::numeric_limits<NumberType>::digits10;
+
+            // the actual conversion
+            const auto written_bytes = snprintf(m_buf.data(), m_buf.size(), "%.*g", d, x);
+
+            // negative value indicates an error
+            assert(written_bytes > 0);
+            // check if buffer was large enough
+            assert(static_cast<size_t>(written_bytes) < m_buf.size());
+
+            // read information from locale
+            const auto loc = localeconv();
+            assert(loc != nullptr);
+            const char thousands_sep = !loc->thousands_sep ? '\0'
+                                       : loc->thousands_sep[0];
+
+            const char decimal_point = !loc->decimal_point ? '\0'
+                                       : loc->decimal_point[0];
+
+            // erase thousands separator
+            if (thousands_sep != '\0')
+            {
+                const auto end = std::remove(m_buf.begin(), m_buf.begin() + written_bytes, thousands_sep);
+                std::fill(end, m_buf.end(), '\0');
+            }
+
+            // convert decimal point to '.'
+            if (decimal_point != '\0' and decimal_point != '.')
+            {
+                for (auto& c : m_buf)
+                {
+                    if (c == decimal_point)
+                    {
+                        c = '.';
+                        break;
+                    }
+                }
+            }
+
+            // determine if need to append ".0"
+            size_t i = 0;
+            bool value_is_int_like = true;
+            for (i = 0; i < m_buf.size(); ++i)
+            {
+                // break when end of number is reached
+                if (m_buf[i] == '\0')
+                {
+                    break;
+                }
+
+                // check if we find non-int character
+                value_is_int_like = value_is_int_like and m_buf[i] != '.' and
+                                    m_buf[i] != 'e' and m_buf[i] != 'E';
+            }
+
+            if (value_is_int_like)
+            {
+                // there must be 2 bytes left for ".0"
+                assert((i + 2) < m_buf.size());
+                // we write to the end of the number
+                assert(m_buf[i] == '\0');
+                assert(m_buf[i - 1] != '\0');
+
+                // add ".0"
+                m_buf[i] = '.';
+                m_buf[i + 1] = '0';
+
+                // the resulting string is properly terminated
+                assert(m_buf[i + 2] == '\0');
+            }
+        }
+    };
+
+
     /*!
     @brief internal implementation of the serialization function
 
@@ -8049,27 +8514,19 @@ class basic_json
 
             case value_t::number_integer:
             {
-                o << m_value.number_integer;
+                o << numtostr(m_value.number_integer).c_str();
                 return;
             }
 
             case value_t::number_unsigned:
             {
-                o << m_value.number_unsigned;
+                o << numtostr(m_value.number_unsigned).c_str();
                 return;
             }
 
             case value_t::number_float:
             {
-                if (m_value.number_float == 0)
-                {
-                    // special case for zero to get "0.0"/"-0.0"
-                    o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0");
-                }
-                else
-                {
-                    o << m_value.number_float;
-                }
+                o << numtostr(m_value.number_float).c_str();
                 return;
             }
 
@@ -8116,6 +8573,11 @@ class basic_json
     class primitive_iterator_t
     {
       public:
+
+        difference_type get_value() const noexcept
+        {
+            return m_it;
+        }
         /// set iterator to a defined beginning
         void set_begin() noexcept
         {
@@ -8140,16 +8602,89 @@ class basic_json
             return (m_it == end_value);
         }
 
-        /// return reference to the value to change and compare
-        operator difference_type& () noexcept
+        friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
         {
-            return m_it;
+            return lhs.m_it == rhs.m_it;
         }
 
-        /// return value to compare
-        constexpr operator difference_type () const noexcept
+        friend constexpr bool operator!=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
         {
-            return m_it;
+            return !(lhs == rhs);
+        }
+
+        friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+        {
+            return lhs.m_it < rhs.m_it;
+        }
+
+        friend constexpr bool operator<=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+        {
+            return lhs.m_it <= rhs.m_it;
+        }
+
+        friend constexpr bool operator>(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+        {
+            return lhs.m_it > rhs.m_it;
+        }
+
+        friend constexpr bool operator>=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+        {
+            return lhs.m_it >= rhs.m_it;
+        }
+
+        primitive_iterator_t operator+(difference_type i)
+        {
+            auto result = *this;
+            result += i;
+            return result;
+        }
+
+        friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+        {
+            return lhs.m_it - rhs.m_it;
+        }
+
+        friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it)
+        {
+            return os << it.m_it;
+        }
+
+        primitive_iterator_t& operator++()
+        {
+            ++m_it;
+            return *this;
+        }
+
+        primitive_iterator_t operator++(int)
+        {
+            auto result = *this;
+            m_it++;
+            return result;
+        }
+
+        primitive_iterator_t& operator--()
+        {
+            --m_it;
+            return *this;
+        }
+
+        primitive_iterator_t operator--(int)
+        {
+            auto result = *this;
+            m_it--;
+            return result;
+        }
+
+        primitive_iterator_t& operator+=(difference_type n)
+        {
+            m_it += n;
+            return *this;
+        }
+
+        primitive_iterator_t& operator-=(difference_type n)
+        {
+            m_it -= n;
+            return *this;
         }
 
       private:
@@ -8501,7 +9036,7 @@ class basic_json
 
                 case basic_json::value_t::null:
                 {
-                    throw std::out_of_range("cannot get value");
+                    JSON_THROW(std::out_of_range("cannot get value"));
                 }
 
                 default:
@@ -8510,10 +9045,8 @@ class basic_json
                     {
                         return *m_object;
                     }
-                    else
-                    {
-                        throw std::out_of_range("cannot get value");
-                    }
+
+                    JSON_THROW(std::out_of_range("cannot get value"));
                 }
             }
         }
@@ -8546,10 +9079,8 @@ class basic_json
                     {
                         return m_object;
                     }
-                    else
-                    {
-                        throw std::out_of_range("cannot get value");
-                    }
+
+                    JSON_THROW(std::out_of_range("cannot get value"));
                 }
             }
         }
@@ -8649,7 +9180,7 @@ class basic_json
             // if objects are not the same, the comparison is undefined
             if (m_object != other.m_object)
             {
-                throw std::domain_error("cannot compare iterators of different containers");
+                JSON_THROW(std::domain_error("cannot compare iterators of different containers"));
             }
 
             assert(m_object != nullptr);
@@ -8691,7 +9222,7 @@ class basic_json
             // if objects are not the same, the comparison is undefined
             if (m_object != other.m_object)
             {
-                throw std::domain_error("cannot compare iterators of different containers");
+                JSON_THROW(std::domain_error("cannot compare iterators of different containers"));
             }
 
             assert(m_object != nullptr);
@@ -8700,7 +9231,7 @@ class basic_json
             {
                 case basic_json::value_t::object:
                 {
-                    throw std::domain_error("cannot compare order of object iterators");
+                    JSON_THROW(std::domain_error("cannot compare order of object iterators"));
                 }
 
                 case basic_json::value_t::array:
@@ -8754,7 +9285,7 @@ class basic_json
             {
                 case basic_json::value_t::object:
                 {
-                    throw std::domain_error("cannot use offsets with object iterators");
+                    JSON_THROW(std::domain_error("cannot use offsets with object iterators"));
                 }
 
                 case basic_json::value_t::array:
@@ -8816,7 +9347,7 @@ class basic_json
             {
                 case basic_json::value_t::object:
                 {
-                    throw std::domain_error("cannot use offsets with object iterators");
+                    JSON_THROW(std::domain_error("cannot use offsets with object iterators"));
                 }
 
                 case basic_json::value_t::array:
@@ -8843,7 +9374,7 @@ class basic_json
             {
                 case basic_json::value_t::object:
                 {
-                    throw std::domain_error("cannot use operator[] for object iterators");
+                    JSON_THROW(std::domain_error("cannot use operator[] for object iterators"));
                 }
 
                 case basic_json::value_t::array:
@@ -8853,19 +9384,17 @@ class basic_json
 
                 case basic_json::value_t::null:
                 {
-                    throw std::out_of_range("cannot get value");
+                    JSON_THROW(std::out_of_range("cannot get value"));
                 }
 
                 default:
                 {
-                    if (m_it.primitive_iterator == -n)
+                    if (m_it.primitive_iterator.get_value() == -n)
                     {
                         return *m_object;
                     }
-                    else
-                    {
-                        throw std::out_of_range("cannot get value");
-                    }
+
+                    JSON_THROW(std::out_of_range("cannot get value"));
                 }
             }
         }
@@ -8882,10 +9411,8 @@ class basic_json
             {
                 return m_it.object_iterator->first;
             }
-            else
-            {
-                throw std::domain_error("cannot use key() for non-object iterators");
-            }
+
+            JSON_THROW(std::domain_error("cannot use key() for non-object iterators"));
         }
 
         /*!
@@ -9040,7 +9567,9 @@ class basic_json
             literal_false,   ///< the `false` literal
             literal_null,    ///< the `null` literal
             value_string,    ///< a string -- use get_string() for actual value
-            value_number,    ///< a number -- use get_number() for actual value
+            value_unsigned,  ///< an unsigned integer -- use get_number() for actual value
+            value_integer,   ///< a signed integer -- use get_number() for actual value
+            value_float,     ///< an floating point number -- use get_number() for actual value
             begin_array,     ///< the character for array begin `[`
             begin_object,    ///< the character for object begin `{`
             end_array,       ///< the character for array end `]`
@@ -9070,7 +9599,7 @@ class basic_json
             // immediately abort if stream is erroneous
             if (s.fail())
             {
-                throw std::invalid_argument("stream error");
+                JSON_THROW(std::invalid_argument("stream error"));
             }
 
             // fill buffer
@@ -9137,7 +9666,7 @@ class basic_json
                 }
                 else
                 {
-                    throw std::invalid_argument("missing or wrong low surrogate");
+                    JSON_THROW(std::invalid_argument("missing or wrong low surrogate"));
                 }
             }
 
@@ -9171,7 +9700,7 @@ class basic_json
             }
             else
             {
-                throw std::out_of_range("code points above 0x10FFFF are invalid");
+                JSON_THROW(std::out_of_range("code points above 0x10FFFF are invalid"));
             }
 
             return result;
@@ -9192,7 +9721,9 @@ class basic_json
                     return "null literal";
                 case token_type::value_string:
                     return "string literal";
-                case token_type::value_number:
+                case lexer::token_type::value_unsigned:
+                case lexer::token_type::value_integer:
+                case lexer::token_type::value_float:
                     return "number literal";
                 case token_type::begin_array:
                     return "'['";
@@ -9465,37 +9996,47 @@ basic_json_parser_12:
                     }
                     if (yych <= '0')
                     {
-                        goto basic_json_parser_13;
+                        goto basic_json_parser_43;
                     }
                     if (yych <= '9')
                     {
-                        goto basic_json_parser_15;
+                        goto basic_json_parser_45;
                     }
                     goto basic_json_parser_5;
 basic_json_parser_13:
                     yyaccept = 1;
                     yych = *(m_marker = ++m_cursor);
-                    if (yych <= 'D')
+                    if (yych <= '9')
                     {
                         if (yych == '.')
                         {
-                            goto basic_json_parser_43;
+                            goto basic_json_parser_47;
+                        }
+                        if (yych >= '0')
+                        {
+                            goto basic_json_parser_48;
                         }
                     }
                     else
                     {
                         if (yych <= 'E')
                         {
-                            goto basic_json_parser_44;
+                            if (yych >= 'E')
+                            {
+                                goto basic_json_parser_51;
+                            }
                         }
-                        if (yych == 'e')
+                        else
                         {
-                            goto basic_json_parser_44;
+                            if (yych == 'e')
+                            {
+                                goto basic_json_parser_51;
+                            }
                         }
                     }
 basic_json_parser_14:
                     {
-                        last_token_type = token_type::value_number;
+                        last_token_type = token_type::value_unsigned;
                         break;
                     }
 basic_json_parser_15:
@@ -9514,7 +10055,7 @@ basic_json_parser_15:
                     {
                         if (yych == '.')
                         {
-                            goto basic_json_parser_43;
+                            goto basic_json_parser_47;
                         }
                         goto basic_json_parser_14;
                     }
@@ -9522,11 +10063,11 @@ basic_json_parser_15:
                     {
                         if (yych <= 'E')
                         {
-                            goto basic_json_parser_44;
+                            goto basic_json_parser_51;
                         }
                         if (yych == 'e')
                         {
-                            goto basic_json_parser_44;
+                            goto basic_json_parser_51;
                         }
                         goto basic_json_parser_14;
                     }
@@ -9553,7 +10094,7 @@ basic_json_parser_23:
                     yych = *(m_marker = ++m_cursor);
                     if (yych == 'a')
                     {
-                        goto basic_json_parser_45;
+                        goto basic_json_parser_52;
                     }
                     goto basic_json_parser_5;
 basic_json_parser_24:
@@ -9561,7 +10102,7 @@ basic_json_parser_24:
                     yych = *(m_marker = ++m_cursor);
                     if (yych == 'u')
                     {
-                        goto basic_json_parser_46;
+                        goto basic_json_parser_53;
                     }
                     goto basic_json_parser_5;
 basic_json_parser_25:
@@ -9569,7 +10110,7 @@ basic_json_parser_25:
                     yych = *(m_marker = ++m_cursor);
                     if (yych == 'r')
                     {
-                        goto basic_json_parser_47;
+                        goto basic_json_parser_54;
                     }
                     goto basic_json_parser_5;
 basic_json_parser_26:
@@ -9651,13 +10192,27 @@ basic_json_parser_31:
                     }
 basic_json_parser_32:
                     m_cursor = m_marker;
-                    if (yyaccept == 0)
+                    if (yyaccept <= 1)
                     {
-                        goto basic_json_parser_5;
+                        if (yyaccept == 0)
+                        {
+                            goto basic_json_parser_5;
+                        }
+                        else
+                        {
+                            goto basic_json_parser_14;
+                        }
                     }
                     else
                     {
-                        goto basic_json_parser_14;
+                        if (yyaccept == 2)
+                        {
+                            goto basic_json_parser_44;
+                        }
+                        else
+                        {
+                            goto basic_json_parser_58;
+                        }
                     }
 basic_json_parser_33:
                     ++m_cursor;
@@ -9738,7 +10293,7 @@ basic_json_parser_35:
                                 }
                                 if (yych <= 'u')
                                 {
-                                    goto basic_json_parser_48;
+                                    goto basic_json_parser_55;
                                 }
                                 goto basic_json_parser_32;
                             }
@@ -9857,64 +10412,159 @@ basic_json_parser_42:
                     }
                     goto basic_json_parser_32;
 basic_json_parser_43:
+                    yyaccept = 2;
+                    yych = *(m_marker = ++m_cursor);
+                    if (yych <= '9')
+                    {
+                        if (yych == '.')
+                        {
+                            goto basic_json_parser_47;
+                        }
+                        if (yych >= '0')
+                        {
+                            goto basic_json_parser_48;
+                        }
+                    }
+                    else
+                    {
+                        if (yych <= 'E')
+                        {
+                            if (yych >= 'E')
+                            {
+                                goto basic_json_parser_51;
+                            }
+                        }
+                        else
+                        {
+                            if (yych == 'e')
+                            {
+                                goto basic_json_parser_51;
+                            }
+                        }
+                    }
+basic_json_parser_44:
+                    {
+                        last_token_type = token_type::value_integer;
+                        break;
+                    }
+basic_json_parser_45:
+                    yyaccept = 2;
+                    m_marker = ++m_cursor;
+                    if ((m_limit - m_cursor) < 3)
+                    {
+                        fill_line_buffer(3);    // LCOV_EXCL_LINE
+                    }
+                    yych = *m_cursor;
+                    if (yych <= '9')
+                    {
+                        if (yych == '.')
+                        {
+                            goto basic_json_parser_47;
+                        }
+                        if (yych <= '/')
+                        {
+                            goto basic_json_parser_44;
+                        }
+                        goto basic_json_parser_45;
+                    }
+                    else
+                    {
+                        if (yych <= 'E')
+                        {
+                            if (yych <= 'D')
+                            {
+                                goto basic_json_parser_44;
+                            }
+                            goto basic_json_parser_51;
+                        }
+                        else
+                        {
+                            if (yych == 'e')
+                            {
+                                goto basic_json_parser_51;
+                            }
+                            goto basic_json_parser_44;
+                        }
+                    }
+basic_json_parser_47:
                     yych = *++m_cursor;
                     if (yych <= '/')
                     {
                         goto basic_json_parser_32;
                     }
                     if (yych <= '9')
-                    {
-                        goto basic_json_parser_49;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_44:
-                    yych = *++m_cursor;
-                    if (yych <= ',')
-                    {
-                        if (yych == '+')
-                        {
-                            goto basic_json_parser_51;
-                        }
-                        goto basic_json_parser_32;
-                    }
-                    else
-                    {
-                        if (yych <= '-')
-                        {
-                            goto basic_json_parser_51;
-                        }
-                        if (yych <= '/')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= '9')
-                        {
-                            goto basic_json_parser_52;
-                        }
-                        goto basic_json_parser_32;
-                    }
-basic_json_parser_45:
-                    yych = *++m_cursor;
-                    if (yych == 'l')
-                    {
-                        goto basic_json_parser_54;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_46:
-                    yych = *++m_cursor;
-                    if (yych == 'l')
-                    {
-                        goto basic_json_parser_55;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_47:
-                    yych = *++m_cursor;
-                    if (yych == 'u')
                     {
                         goto basic_json_parser_56;
                     }
                     goto basic_json_parser_32;
 basic_json_parser_48:
+                    ++m_cursor;
+                    if (m_limit <= m_cursor)
+                    {
+                        fill_line_buffer(1);    // LCOV_EXCL_LINE
+                    }
+                    yych = *m_cursor;
+                    if (yych <= '/')
+                    {
+                        goto basic_json_parser_50;
+                    }
+                    if (yych <= '9')
+                    {
+                        goto basic_json_parser_48;
+                    }
+basic_json_parser_50:
+                    {
+                        last_token_type = token_type::parse_error;
+                        break;
+                    }
+basic_json_parser_51:
+                    yych = *++m_cursor;
+                    if (yych <= ',')
+                    {
+                        if (yych == '+')
+                        {
+                            goto basic_json_parser_59;
+                        }
+                        goto basic_json_parser_32;
+                    }
+                    else
+                    {
+                        if (yych <= '-')
+                        {
+                            goto basic_json_parser_59;
+                        }
+                        if (yych <= '/')
+                        {
+                            goto basic_json_parser_32;
+                        }
+                        if (yych <= '9')
+                        {
+                            goto basic_json_parser_60;
+                        }
+                        goto basic_json_parser_32;
+                    }
+basic_json_parser_52:
+                    yych = *++m_cursor;
+                    if (yych == 'l')
+                    {
+                        goto basic_json_parser_62;
+                    }
+                    goto basic_json_parser_32;
+basic_json_parser_53:
+                    yych = *++m_cursor;
+                    if (yych == 'l')
+                    {
+                        goto basic_json_parser_63;
+                    }
+                    goto basic_json_parser_32;
+basic_json_parser_54:
+                    yych = *++m_cursor;
+                    if (yych == 'u')
+                    {
+                        goto basic_json_parser_64;
+                    }
+                    goto basic_json_parser_32;
+basic_json_parser_55:
                     ++m_cursor;
                     if (m_limit <= m_cursor)
                     {
@@ -9929,7 +10579,7 @@ basic_json_parser_48:
                         }
                         if (yych <= '9')
                         {
-                            goto basic_json_parser_57;
+                            goto basic_json_parser_65;
                         }
                         goto basic_json_parser_32;
                     }
@@ -9937,7 +10587,7 @@ basic_json_parser_48:
                     {
                         if (yych <= 'F')
                         {
-                            goto basic_json_parser_57;
+                            goto basic_json_parser_65;
                         }
                         if (yych <= '`')
                         {
@@ -9945,12 +10595,12 @@ basic_json_parser_48:
                         }
                         if (yych <= 'f')
                         {
-                            goto basic_json_parser_57;
+                            goto basic_json_parser_65;
                         }
                         goto basic_json_parser_32;
                     }
-basic_json_parser_49:
-                    yyaccept = 1;
+basic_json_parser_56:
+                    yyaccept = 3;
                     m_marker = ++m_cursor;
                     if ((m_limit - m_cursor) < 3)
                     {
@@ -9961,27 +10611,30 @@ basic_json_parser_49:
                     {
                         if (yych <= '/')
                         {
-                            goto basic_json_parser_14;
+                            goto basic_json_parser_58;
                         }
                         if (yych <= '9')
                         {
-                            goto basic_json_parser_49;
+                            goto basic_json_parser_56;
                         }
-                        goto basic_json_parser_14;
                     }
                     else
                     {
                         if (yych <= 'E')
                         {
-                            goto basic_json_parser_44;
+                            goto basic_json_parser_51;
                         }
                         if (yych == 'e')
                         {
-                            goto basic_json_parser_44;
+                            goto basic_json_parser_51;
                         }
-                        goto basic_json_parser_14;
                     }
-basic_json_parser_51:
+basic_json_parser_58:
+                    {
+                        last_token_type = token_type::value_float;
+                        break;
+                    }
+basic_json_parser_59:
                     yych = *++m_cursor;
                     if (yych <= '/')
                     {
@@ -9991,7 +10644,7 @@ basic_json_parser_51:
                     {
                         goto basic_json_parser_32;
                     }
-basic_json_parser_52:
+basic_json_parser_60:
                     ++m_cursor;
                     if (m_limit <= m_cursor)
                     {
@@ -10000,35 +10653,35 @@ basic_json_parser_52:
                     yych = *m_cursor;
                     if (yych <= '/')
                     {
-                        goto basic_json_parser_14;
+                        goto basic_json_parser_58;
                     }
                     if (yych <= '9')
                     {
-                        goto basic_json_parser_52;
+                        goto basic_json_parser_60;
                     }
-                    goto basic_json_parser_14;
-basic_json_parser_54:
+                    goto basic_json_parser_58;
+basic_json_parser_62:
                     yych = *++m_cursor;
                     if (yych == 's')
                     {
-                        goto basic_json_parser_58;
+                        goto basic_json_parser_66;
                     }
                     goto basic_json_parser_32;
-basic_json_parser_55:
+basic_json_parser_63:
                     yych = *++m_cursor;
                     if (yych == 'l')
                     {
-                        goto basic_json_parser_59;
+                        goto basic_json_parser_67;
                     }
                     goto basic_json_parser_32;
-basic_json_parser_56:
+basic_json_parser_64:
                     yych = *++m_cursor;
                     if (yych == 'e')
                     {
-                        goto basic_json_parser_61;
+                        goto basic_json_parser_69;
                     }
                     goto basic_json_parser_32;
-basic_json_parser_57:
+basic_json_parser_65:
                     ++m_cursor;
                     if (m_limit <= m_cursor)
                     {
@@ -10043,7 +10696,7 @@ basic_json_parser_57:
                         }
                         if (yych <= '9')
                         {
-                            goto basic_json_parser_63;
+                            goto basic_json_parser_71;
                         }
                         goto basic_json_parser_32;
                     }
@@ -10051,7 +10704,7 @@ basic_json_parser_57:
                     {
                         if (yych <= 'F')
                         {
-                            goto basic_json_parser_63;
+                            goto basic_json_parser_71;
                         }
                         if (yych <= '`')
                         {
@@ -10059,30 +10712,30 @@ basic_json_parser_57:
                         }
                         if (yych <= 'f')
                         {
-                            goto basic_json_parser_63;
+                            goto basic_json_parser_71;
                         }
                         goto basic_json_parser_32;
                     }
-basic_json_parser_58:
+basic_json_parser_66:
                     yych = *++m_cursor;
                     if (yych == 'e')
                     {
-                        goto basic_json_parser_64;
+                        goto basic_json_parser_72;
                     }
                     goto basic_json_parser_32;
-basic_json_parser_59:
+basic_json_parser_67:
                     ++m_cursor;
                     {
                         last_token_type = token_type::literal_null;
                         break;
                     }
-basic_json_parser_61:
+basic_json_parser_69:
                     ++m_cursor;
                     {
                         last_token_type = token_type::literal_true;
                         break;
                     }
-basic_json_parser_63:
+basic_json_parser_71:
                     ++m_cursor;
                     if (m_limit <= m_cursor)
                     {
@@ -10097,7 +10750,7 @@ basic_json_parser_63:
                         }
                         if (yych <= '9')
                         {
-                            goto basic_json_parser_66;
+                            goto basic_json_parser_74;
                         }
                         goto basic_json_parser_32;
                     }
@@ -10105,7 +10758,7 @@ basic_json_parser_63:
                     {
                         if (yych <= 'F')
                         {
-                            goto basic_json_parser_66;
+                            goto basic_json_parser_74;
                         }
                         if (yych <= '`')
                         {
@@ -10113,17 +10766,17 @@ basic_json_parser_63:
                         }
                         if (yych <= 'f')
                         {
-                            goto basic_json_parser_66;
+                            goto basic_json_parser_74;
                         }
                         goto basic_json_parser_32;
                     }
-basic_json_parser_64:
+basic_json_parser_72:
                     ++m_cursor;
                     {
                         last_token_type = token_type::literal_false;
                         break;
                     }
-basic_json_parser_66:
+basic_json_parser_74:
                     ++m_cursor;
                     if (m_limit <= m_cursor)
                     {
@@ -10210,7 +10863,7 @@ basic_json_parser_66:
             assert(m_marker == nullptr or m_marker  <= m_limit);
 
             // number of processed characters (p)
-            const size_t num_processed_chars = static_cast<size_t>(m_start - m_content);
+            const auto num_processed_chars = static_cast<size_t>(m_start - m_content);
             // offset for m_marker wrt. to m_start
             const auto offset_marker = (m_marker == nullptr) ? 0 : m_marker - m_start;
             // number of unprocessed characters (u)
@@ -10220,7 +10873,7 @@ basic_json_parser_66:
             if (m_stream == nullptr or m_stream->eof())
             {
                 // m_start may or may not be pointing into m_line_buffer at
-                // this point. We trust the standand library to do the right
+                // this point. We trust the standard library to do the right
                 // thing. See http://stackoverflow.com/q/28142011/266378
                 m_line_buffer.assign(m_start, m_limit);
 
@@ -10308,7 +10961,7 @@ basic_json_parser_66:
         m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This
         can be rephrased to m_cursor - m_start - 2 > x. With the
         precondition, we x <= 0, meaning that the loop condition holds
-        indefinitly if i is always decreased. However, observe that the value
+        indefinitely if i is always decreased. However, observe that the value
         of i is strictly increasing with each iteration, as it is incremented
         by 1 in the iteration expression and never decremented inside the loop
         body. Hence, the loop condition will eventually be false which
@@ -10403,7 +11056,7 @@ basic_json_parser_66:
                                 // make sure there is a subsequent unicode
                                 if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u')
                                 {
-                                    throw std::invalid_argument("missing low surrogate");
+                                    JSON_THROW(std::invalid_argument("missing low surrogate"));
                                 }
 
                                 // get code yyyy from uxxxx\uyyyy
@@ -10416,7 +11069,7 @@ basic_json_parser_66:
                             else if (codepoint >= 0xDC00 and codepoint <= 0xDFFF)
                             {
                                 // we found a lone low surrogate
-                                throw std::invalid_argument("missing high surrogate");
+                                JSON_THROW(std::invalid_argument("missing high surrogate"));
                             }
                             else
                             {
@@ -10434,59 +11087,155 @@ basic_json_parser_66:
             return result;
         }
 
-        /*!
-        @brief parse floating point number
-
-        This function (and its overloads) serves to select the most approprate
-        standard floating point number parsing function based on the type
-        supplied via the first parameter.  Set this to @a
-        static_cast<number_float_t*>(nullptr).
-
-        @param[in,out] endptr recieves a pointer to the first character after
-        the number
-
-        @return the floating point number
-        */
-        long double str_to_float_t(long double* /* type */, char** endptr) const
-        {
-            return std::strtold(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
-        }
 
         /*!
-        @brief parse floating point number
+        @brief parse string into a built-in arithmetic type as if the current
+               locale is POSIX.
 
-        This function (and its overloads) serves to select the most approprate
-        standard floating point number parsing function based on the type
-        supplied via the first parameter.  Set this to @a
-        static_cast<number_float_t*>(nullptr).
+        @note in floating-point case strtod may parse past the token's end -
+              this is not an error
 
-        @param[in,out] endptr  recieves a pointer to the first character after
-        the number
-
-        @return the floating point number
+        @note any leading blanks are not handled
         */
-        double str_to_float_t(double* /* type */, char** endptr) const
+        struct strtonum
         {
-            return std::strtod(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
-        }
+          public:
+            strtonum(const char* start, const char* end)
+                : m_start(start), m_end(end)
+            {}
 
-        /*!
-        @brief parse floating point number
+            /*!
+            @return true iff parsed successfully as number of type T
 
-        This function (and its overloads) serves to select the most approprate
-        standard floating point number parsing function based on the type
-        supplied via the first parameter.  Set this to @a
-        static_cast<number_float_t*>(nullptr).
+            @param[in,out] val shall contain parsed value, or undefined value
+            if could not parse
+            */
+            template<typename T, typename = typename std::enable_if<std::is_arithmetic<T>::value>::type>
+            bool to(T& val) const
+            {
+                return parse(val, std::is_integral<T>());
+            }
 
-        @param[in,out] endptr  recieves a pointer to the first character after
-        the number
+          private:
+            const char* const m_start = nullptr;
+            const char* const m_end = nullptr;
 
-        @return the floating point number
-        */
-        float str_to_float_t(float* /* type */, char** endptr) const
-        {
-            return std::strtof(reinterpret_cast<typename string_t::const_pointer>(m_start), endptr);
-        }
+            // floating-point conversion
+
+            // overloaded wrappers for strtod/strtof/strtold
+            // that will be called from parse<floating_point_t>
+            static void strtof(float& f, const char* str, char** endptr)
+            {
+                f = std::strtof(str, endptr);
+            }
+
+            static void strtof(double& f, const char* str, char** endptr)
+            {
+                f = std::strtod(str, endptr);
+            }
+
+            static void strtof(long double& f, const char* str, char** endptr)
+            {
+                f = std::strtold(str, endptr);
+            }
+
+            template<typename T>
+            bool parse(T& value, /*is_integral=*/std::false_type) const
+            {
+                // replace decimal separator with locale-specific version,
+                // when necessary; data will point to either the original
+                // string, or buf, or tempstr containing the fixed string.
+                std::string tempstr;
+                std::array<char, 64> buf;
+                const size_t len = static_cast<size_t>(m_end - m_start);
+
+                // lexer will reject empty numbers
+                assert(len > 0);
+
+                // since dealing with strtod family of functions, we're
+                // getting the decimal point char from the C locale facilities
+                // instead of C++'s numpunct facet of the current std::locale
+                const auto loc = localeconv();
+                assert(loc != nullptr);
+                const char decimal_point_char = (loc->decimal_point == nullptr) ? '.' : loc->decimal_point[0];
+
+                const char* data = m_start;
+
+                if (decimal_point_char != '.')
+                {
+                    const size_t ds_pos = static_cast<size_t>(std::find(m_start, m_end, '.') - m_start);
+
+                    if (ds_pos != len)
+                    {
+                        // copy the data into the local buffer or tempstr, if
+                        // buffer is too small; replace decimal separator, and
+                        // update data to point to the modified bytes
+                        if ((len + 1) < buf.size())
+                        {
+                            std::copy(m_start, m_end, buf.begin());
+                            buf[len] = 0;
+                            buf[ds_pos] = decimal_point_char;
+                            data = buf.data();
+                        }
+                        else
+                        {
+                            tempstr.assign(m_start, m_end);
+                            tempstr[ds_pos] = decimal_point_char;
+                            data = tempstr.c_str();
+                        }
+                    }
+                }
+
+                char* endptr = nullptr;
+                value = 0;
+                // this calls appropriate overload depending on T
+                strtof(value, data, &endptr);
+
+                // parsing was successful iff strtof parsed exactly the number
+                // of characters determined by the lexer (len)
+                const bool ok = (endptr == (data + len));
+
+                if (ok and (value == static_cast<T>(0.0)) and (*data == '-'))
+                {
+                    // some implementations forget to negate the zero
+                    value = -0.0;
+                }
+
+                return ok;
+            }
+
+            // integral conversion
+
+            signed long long parse_integral(char** endptr, /*is_signed*/std::true_type) const
+            {
+                return std::strtoll(m_start, endptr, 10);
+            }
+
+            unsigned long long parse_integral(char** endptr, /*is_signed*/std::false_type) const
+            {
+                return std::strtoull(m_start, endptr, 10);
+            }
+
+            template<typename T>
+            bool parse(T& value, /*is_integral=*/std::true_type) const
+            {
+                char* endptr = nullptr;
+                errno = 0; // these are thread-local
+                const auto x = parse_integral(&endptr, std::is_signed<T>());
+
+                // called right overload?
+                static_assert(std::is_signed<T>() == std::is_signed<decltype(x)>(), "");
+
+                value = static_cast<T>(x);
+
+                return (x == static_cast<decltype(x)>(value)) // x fits into destination T
+                       and (x < 0) == (value < 0)             // preserved sign
+                       //and ((x != 0) or is_integral())        // strto[u]ll did nto fail
+                       and (errno == 0)                       // strto[u]ll did not overflow
+                       and (m_start < m_end)                  // token was not empty
+                       and (endptr == m_end);                 // parsed entire token exactly
+            }
+        };
 
         /*!
         @brief return number value for number tokens
@@ -10495,125 +11244,84 @@ basic_json_parser_66:
         number type (either integer, unsigned integer or floating point),
         which is passed back to the caller via the result parameter.
 
-        This function parses the integer component up to the radix point or
-        exponent while collecting information about the 'floating point
-        representation', which it stores in the result parameter. If there is
-        no radix point or exponent, and the number can fit into a @ref
-        number_integer_t or @ref number_unsigned_t then it sets the result
-        parameter accordingly.
+        integral numbers that don't fit into the the range of the respective
+        type are parsed as number_float_t
 
-        If the number is a floating point number the number is then parsed
-        using @a std:strtod (or @a std:strtof or @a std::strtold).
+        floating-point values do not satisfy std::isfinite predicate
+        are converted to value_t::null
 
-        @param[out] result  @ref basic_json object to receive the number, or
-        NAN if the conversion read past the current token. The latter case
-        needs to be treated by the caller function.
+        throws if the entire string [m_start .. m_cursor) cannot be
+        interpreted as a number
+
+        @param[out] result  @ref basic_json object to receive the number.
+        @param[in]  token   the type of the number token
         */
-        void get_number(basic_json& result) const
+        bool get_number(basic_json& result, const token_type token) const
         {
             assert(m_start != nullptr);
+            assert(m_start < m_cursor);
+            assert((token == token_type::value_unsigned) or
+                   (token == token_type::value_integer) or
+                   (token == token_type::value_float));
 
-            const lexer::lexer_char_t* curptr = m_start;
+            strtonum num_converter(reinterpret_cast<const char*>(m_start),
+                                   reinterpret_cast<const char*>(m_cursor));
 
-            // accumulate the integer conversion result (unsigned for now)
-            number_unsigned_t value = 0;
-
-            // maximum absolute value of the relevant integer type
-            number_unsigned_t max;
-
-            // temporarily store the type to avoid unecessary bitfield access
-            value_t type;
-
-            // look for sign
-            if (*curptr == '-')
+            switch (token)
             {
-                type = value_t::number_integer;
-                max = static_cast<uint64_t>((std::numeric_limits<number_integer_t>::max)()) + 1;
-                curptr++;
-            }
-            else
-            {
-                type = value_t::number_unsigned;
-                max = static_cast<uint64_t>((std::numeric_limits<number_unsigned_t>::max)());
-            }
-
-            // count the significant figures
-            for (; curptr < m_cursor; curptr++)
-            {
-                // quickly skip tests if a digit
-                if (*curptr < '0' || *curptr > '9')
+                case lexer::token_type::value_unsigned:
                 {
-                    if (*curptr == '.')
+                    number_unsigned_t val;
+                    if (num_converter.to(val))
                     {
-                        // don't count '.' but change to float
-                        type = value_t::number_float;
-                        continue;
+                        // parsing successful
+                        result.m_type = value_t::number_unsigned;
+                        result.m_value = val;
+                        return true;
                     }
-                    // assume exponent (if not then will fail parse): change to
-                    // float, stop counting and record exponent details
-                    type = value_t::number_float;
                     break;
                 }
 
-                // skip if definitely not an integer
-                if (type != value_t::number_float)
+                case lexer::token_type::value_integer:
                 {
-                    auto digit = static_cast<number_unsigned_t>(*curptr - '0');
-
-                    // overflow if value * 10 + digit > max, move terms around
-                    // to avoid overflow in intermediate values
-                    if (value > (max - digit) / 10)
+                    number_integer_t val;
+                    if (num_converter.to(val))
                     {
-                        // overflow
-                        type = value_t::number_float;
-                    }
-                    else
-                    {
-                        // no overflow
-                        value = value * 10 + digit;
+                        // parsing successful
+                        result.m_type = value_t::number_integer;
+                        result.m_value = val;
+                        return true;
                     }
+                    break;
+                }
+
+                default:
+                {
+                    break;
                 }
             }
 
-            // save the value (if not a float)
-            if (type == value_t::number_unsigned)
+            // parse float (either explicitly or because a previous conversion
+            // failed)
+            number_float_t val;
+            if (num_converter.to(val))
             {
-                result.m_value.number_unsigned = value;
-            }
-            else if (type == value_t::number_integer)
-            {
-                // invariant: if we parsed a '-', the absolute value is between
-                // 0 (we allow -0) and max == -INT64_MIN
-                assert(value >= 0);
-                assert(value <= max);
-
-                if (value == max)
-                {
-                    // we cannot simply negate value (== max == -INT64_MIN),
-                    // see https://github.com/nlohmann/json/issues/389
-                    result.m_value.number_integer = static_cast<number_integer_t>(INT64_MIN);
-                }
-                else
-                {
-                    // all other values can be negated safely
-                    result.m_value.number_integer = -static_cast<number_integer_t>(value);
-                }
-            }
-            else
-            {
-                // parse with strtod
-                result.m_value.number_float = str_to_float_t(static_cast<number_float_t*>(nullptr), NULL);
+                // parsing successful
+                result.m_type = value_t::number_float;
+                result.m_value = val;
 
                 // replace infinity and NAN by null
                 if (not std::isfinite(result.m_value.number_float))
                 {
-                    type = value_t::null;
+                    result.m_type  = value_t::null;
                     result.m_value = basic_json::json_value();
                 }
+
+                return true;
             }
 
-            // save the type
-            result.m_type = type;
+            // couldn't parse number in any format
+            return false;
         }
 
       private:
@@ -10857,9 +11565,11 @@ basic_json_parser_66:
                     break;
                 }
 
-                case lexer::token_type::value_number:
+                case lexer::token_type::value_unsigned:
+                case lexer::token_type::value_integer:
+                case lexer::token_type::value_float:
                 {
-                    m_lexer.get_number(result);
+                    m_lexer.get_number(result, last_token);
                     get_token();
                     break;
                 }
@@ -10894,7 +11604,7 @@ basic_json_parser_66:
                               "'") :
                               lexer::token_type_name(last_token));
                 error_msg += "; expected " + lexer::token_type_name(t);
-                throw std::invalid_argument(error_msg);
+                JSON_THROW(std::invalid_argument(error_msg));
             }
         }
 
@@ -10906,7 +11616,7 @@ basic_json_parser_66:
                 error_msg += (last_token == lexer::token_type::parse_error ? ("'" +  m_lexer.get_token_string() +
                               "'") :
                               lexer::token_type_name(last_token));
-                throw std::invalid_argument(error_msg);
+                JSON_THROW(std::invalid_argument(error_msg));
             }
         }
 
@@ -11002,7 +11712,7 @@ basic_json_parser_66:
         {
             if (is_root())
             {
-                throw std::domain_error("JSON pointer has no parent");
+                JSON_THROW(std::domain_error("JSON pointer has no parent"));
             }
 
             auto last = reference_tokens.back();
@@ -11020,7 +11730,7 @@ basic_json_parser_66:
         {
             if (is_root())
             {
-                throw std::domain_error("JSON pointer has no parent");
+                JSON_THROW(std::domain_error("JSON pointer has no parent"));
             }
 
             json_pointer result = *this;
@@ -11081,7 +11791,7 @@ basic_json_parser_66:
                     */
                     default:
                     {
-                        throw std::domain_error("invalid value to unflatten");
+                        JSON_THROW(std::domain_error("invalid value to unflatten"));
                     }
                 }
             }
@@ -11149,12 +11859,12 @@ basic_json_parser_66:
                         // error condition (cf. RFC 6901, Sect. 4)
                         if (reference_token.size() > 1 and reference_token[0] == '0')
                         {
-                            throw std::domain_error("array index must not begin with '0'");
+                            JSON_THROW(std::domain_error("array index must not begin with '0'"));
                         }
 
                         if (reference_token == "-")
                         {
-                            // explicityly treat "-" as index beyond the end
+                            // explicitly treat "-" as index beyond the end
                             ptr = &ptr->operator[](ptr->m_value.array->size());
                         }
                         else
@@ -11167,7 +11877,7 @@ basic_json_parser_66:
 
                     default:
                     {
-                        throw std::out_of_range("unresolved reference token '" + reference_token + "'");
+                        JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'"));
                     }
                 }
             }
@@ -11193,15 +11903,15 @@ basic_json_parser_66:
                         if (reference_token == "-")
                         {
                             // "-" always fails the range check
-                            throw std::out_of_range("array index '-' (" +
-                                                    std::to_string(ptr->m_value.array->size()) +
-                                                    ") is out of range");
+                            JSON_THROW(std::out_of_range("array index '-' (" +
+                                                         std::to_string(ptr->m_value.array->size()) +
+                                                         ") is out of range"));
                         }
 
                         // error condition (cf. RFC 6901, Sect. 4)
                         if (reference_token.size() > 1 and reference_token[0] == '0')
                         {
-                            throw std::domain_error("array index must not begin with '0'");
+                            JSON_THROW(std::domain_error("array index must not begin with '0'"));
                         }
 
                         // note: at performs range check
@@ -11211,7 +11921,7 @@ basic_json_parser_66:
 
                     default:
                     {
-                        throw std::out_of_range("unresolved reference token '" + reference_token + "'");
+                        JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'"));
                     }
                 }
             }
@@ -11245,15 +11955,15 @@ basic_json_parser_66:
                         if (reference_token == "-")
                         {
                             // "-" cannot be used for const access
-                            throw std::out_of_range("array index '-' (" +
-                                                    std::to_string(ptr->m_value.array->size()) +
-                                                    ") is out of range");
+                            JSON_THROW(std::out_of_range("array index '-' (" +
+                                                         std::to_string(ptr->m_value.array->size()) +
+                                                         ") is out of range"));
                         }
 
                         // error condition (cf. RFC 6901, Sect. 4)
                         if (reference_token.size() > 1 and reference_token[0] == '0')
                         {
-                            throw std::domain_error("array index must not begin with '0'");
+                            JSON_THROW(std::domain_error("array index must not begin with '0'"));
                         }
 
                         // use unchecked array access
@@ -11263,7 +11973,7 @@ basic_json_parser_66:
 
                     default:
                     {
-                        throw std::out_of_range("unresolved reference token '" + reference_token + "'");
+                        JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'"));
                     }
                 }
             }
@@ -11289,15 +11999,15 @@ basic_json_parser_66:
                         if (reference_token == "-")
                         {
                             // "-" always fails the range check
-                            throw std::out_of_range("array index '-' (" +
-                                                    std::to_string(ptr->m_value.array->size()) +
-                                                    ") is out of range");
+                            JSON_THROW(std::out_of_range("array index '-' (" +
+                                                         std::to_string(ptr->m_value.array->size()) +
+                                                         ") is out of range"));
                         }
 
                         // error condition (cf. RFC 6901, Sect. 4)
                         if (reference_token.size() > 1 and reference_token[0] == '0')
                         {
-                            throw std::domain_error("array index must not begin with '0'");
+                            JSON_THROW(std::domain_error("array index must not begin with '0'"));
                         }
 
                         // note: at performs range check
@@ -11307,7 +12017,7 @@ basic_json_parser_66:
 
                     default:
                     {
-                        throw std::out_of_range("unresolved reference token '" + reference_token + "'");
+                        JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'"));
                     }
                 }
             }
@@ -11329,7 +12039,7 @@ basic_json_parser_66:
             // check if nonempty reference string begins with slash
             if (reference_string[0] != '/')
             {
-                throw std::domain_error("JSON pointer must be empty or begin with '/'");
+                JSON_THROW(std::domain_error("JSON pointer must be empty or begin with '/'"));
             }
 
             // extract the reference tokens:
@@ -11337,7 +12047,7 @@ basic_json_parser_66:
             // - start: position after the previous slash
             for (
                 // search for the first slash after the first character
-                size_t slash = reference_string.find_first_of("/", 1),
+                size_t slash = reference_string.find_first_of('/', 1),
                 // set the beginning of the first reference token
                 start = 1;
                 // we can stop if start == string::npos+1 = 0
@@ -11346,16 +12056,16 @@ basic_json_parser_66:
                 // (will eventually be 0 if slash == std::string::npos)
                 start = slash + 1,
                 // find next slash
-                slash = reference_string.find_first_of("/", start))
+                slash = reference_string.find_first_of('/', start))
             {
                 // use the text between the beginning of the reference token
                 // (start) and the last slash (slash).
                 auto reference_token = reference_string.substr(start, slash - start);
 
                 // check reference tokens are properly escaped
-                for (size_t pos = reference_token.find_first_of("~");
+                for (size_t pos = reference_token.find_first_of('~');
                         pos != std::string::npos;
-                        pos = reference_token.find_first_of("~", pos + 1))
+                        pos = reference_token.find_first_of('~', pos + 1))
                 {
                     assert(reference_token[pos] == '~');
 
@@ -11364,7 +12074,7 @@ basic_json_parser_66:
                             (reference_token[pos + 1] != '0' and
                              reference_token[pos + 1] != '1'))
                     {
-                        throw std::domain_error("escape error: '~' must be followed with '0' or '1'");
+                        JSON_THROW(std::domain_error("escape error: '~' must be followed with '0' or '1'"));
                     }
                 }
 
@@ -11490,7 +12200,7 @@ basic_json_parser_66:
         {
             if (not value.is_object())
             {
-                throw std::domain_error("only objects can be unflattened");
+                JSON_THROW(std::domain_error("only objects can be unflattened"));
             }
 
             basic_json result;
@@ -11500,7 +12210,7 @@ basic_json_parser_66:
             {
                 if (not element.second.is_primitive())
                 {
-                    throw std::domain_error("values in object must be primitive");
+                    JSON_THROW(std::domain_error("values in object must be primitive"));
                 }
 
                 // assign value to reference pointed to by JSON pointer; Note
@@ -11515,6 +12225,18 @@ basic_json_parser_66:
         }
 
       private:
+        friend bool operator==(json_pointer const& lhs,
+                               json_pointer const& rhs) noexcept
+        {
+            return lhs.reference_tokens == rhs.reference_tokens;
+        }
+
+        friend bool operator!=(json_pointer const& lhs,
+                               json_pointer const& rhs) noexcept
+        {
+            return !(lhs == rhs);
+        }
+
         /// the reference tokens
         std::vector<std::string> reference_tokens {};
     };
@@ -11649,7 +12371,7 @@ basic_json_parser_66:
     primitive. The original JSON value can be restored using the @ref
     unflatten() function.
 
-    @return an object that maps JSON pointers to primitve values
+    @return an object that maps JSON pointers to primitive values
 
     @note Empty objects and arrays are flattened to `null` and will not be
           reconstructed correctly by the @ref unflatten() function.
@@ -11716,7 +12438,7 @@ basic_json_parser_66:
 
     [JSON Patch](http://jsonpatch.com) defines a JSON document structure for
     expressing a sequence of operations to apply to a JSON) document. With
-    this funcion, a JSON Patch is applied to the current JSON value by
+    this function, a JSON Patch is applied to the current JSON value by
     executing all operations from the patch.
 
     @param[in] json_patch  JSON patch document
@@ -11829,7 +12551,7 @@ basic_json_parser_66:
                             if (static_cast<size_type>(idx) > parent.size())
                             {
                                 // avoid undefined behavior
-                                throw std::out_of_range("array index " + std::to_string(idx) + " is out of range");
+                                JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range"));
                             }
                             else
                             {
@@ -11867,7 +12589,7 @@ basic_json_parser_66:
                 }
                 else
                 {
-                    throw std::out_of_range("key '" + last_path + "' not found");
+                    JSON_THROW(std::out_of_range("key '" + last_path + "' not found"));
                 }
             }
             else if (parent.is_array())
@@ -11881,10 +12603,10 @@ basic_json_parser_66:
         if (not json_patch.is_array())
         {
             // a JSON patch must be an array of objects
-            throw std::invalid_argument("JSON patch must be an array of objects");
+            JSON_THROW(std::invalid_argument("JSON patch must be an array of objects"));
         }
 
-        // iterate and apply th eoperations
+        // iterate and apply the operations
         for (const auto& val : json_patch)
         {
             // wrapper to get a value for an operation
@@ -11901,13 +12623,13 @@ basic_json_parser_66:
                 // check if desired value is present
                 if (it == val.m_value.object->end())
                 {
-                    throw std::invalid_argument(error_msg + " must have member '" + member + "'");
+                    JSON_THROW(std::invalid_argument(error_msg + " must have member '" + member + "'"));
                 }
 
                 // check if result is of type string
                 if (string_type and not it->second.is_string())
                 {
-                    throw std::invalid_argument(error_msg + " must have string member '" + member + "'");
+                    JSON_THROW(std::invalid_argument(error_msg + " must have string member '" + member + "'"));
                 }
 
                 // no error: return value
@@ -11917,7 +12639,7 @@ basic_json_parser_66:
             // type check
             if (not val.is_object())
             {
-                throw std::invalid_argument("JSON patch must be an array of objects");
+                JSON_THROW(std::invalid_argument("JSON patch must be an array of objects"));
             }
 
             // collect mandatory members
@@ -11976,13 +12698,13 @@ basic_json_parser_66:
                 case patch_operations::test:
                 {
                     bool success = false;
-                    try
+                    JSON_TRY
                     {
                         // check if "value" matches the one at "path"
                         // the "path" location must exist - use at()
                         success = (result.at(ptr) == get_value("test", "value", false));
                     }
-                    catch (std::out_of_range&)
+                    JSON_CATCH (std::out_of_range&)
                     {
                         // ignore out of range errors: success remains false
                     }
@@ -11990,7 +12712,7 @@ basic_json_parser_66:
                     // throw an exception if test fails
                     if (not success)
                     {
-                        throw std::domain_error("unsuccessful: " + val.dump());
+                        JSON_THROW(std::domain_error("unsuccessful: " + val.dump()));
                     }
 
                     break;
@@ -12000,7 +12722,7 @@ basic_json_parser_66:
                 {
                     // op must be "add", "remove", "replace", "move", "copy", or
                     // "test"
-                    throw std::invalid_argument("operation value '" + op + "' is invalid");
+                    JSON_THROW(std::invalid_argument("operation value '" + op + "' is invalid"));
                 }
             }
         }
@@ -12023,8 +12745,8 @@ basic_json_parser_66:
     @note Currently, only `remove`, `add`, and `replace` operations are
           generated.
 
-    @param[in] source  JSON value to copare from
-    @param[in] target  JSON value to copare against
+    @param[in] source  JSON value to compare from
+    @param[in] target  JSON value to compare against
     @param[in] path    helper value to create JSON pointers
 
     @return a JSON patch to convert the @a source to @a target
@@ -12175,7 +12897,6 @@ basic_json_parser_66:
     /// @}
 };
 
-
 /////////////
 // presets //
 /////////////
@@ -12189,7 +12910,7 @@ uses the standard template types.
 @since version 1.0.0
 */
 using json = basic_json<>;
-}
+} // namespace nlohmann
 
 
 ///////////////////////
@@ -12230,7 +12951,7 @@ struct hash<nlohmann::json>
         return h(j.dump());
     }
 };
-}
+} // namespace std
 
 /*!
 @brief user-defined string literal for JSON values
@@ -12273,4 +12994,10 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std
     #pragma GCC diagnostic pop
 #endif
 
+// clean up
+#undef JSON_CATCH
+#undef JSON_DEPRECATED
+#undef JSON_THROW
+#undef JSON_TRY
+
 #endif

From b672717096aa5940a99cc5ddf0aa23974de06bbe Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Mon, 19 Jun 2017 14:38:39 +0100
Subject: [PATCH 002/115] Test_serialiation update for JSON

---
 tests/IO/Test_serialisation.cc | 93 ++++++++++++++++++----------------
 1 file changed, 49 insertions(+), 44 deletions(-)

diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc
index d5b52044..3988784c 100644
--- a/tests/IO/Test_serialisation.cc
+++ b/tests/IO/Test_serialisation.cc
@@ -151,6 +151,11 @@ int main(int argc,char **argv)
   ioTest<TextWriter, TextReader>("iotest.dat", obj, "text   (object)           ");
   ioTest<TextWriter, TextReader>("iotest.dat", vec, "text   (vector of objects)");
   ioTest<TextWriter, TextReader>("iotest.dat", pair, "text   (pair of objects)");
+  //// text
+  ioTest<JSONWriter, JSONReader>("iotest.json", obj, "JSON   (object)           ");
+  ioTest<JSONWriter, JSONReader>("iotest.json", vec, "JSON   (vector of objects)");
+  ioTest<JSONWriter, JSONReader>("iotest.json", pair, "JSON   (pair of objects)");
+
   //// HDF5
 #undef HAVE_HDF5
 #ifdef HAVE_HDF5
@@ -194,50 +199,50 @@ int main(int argc,char **argv)
   std::cout << std::endl;
 
 
-  std::cout << ".:::::: Testing JSON classes "<< std::endl;
-
-
-  {
-    JSONWriter JW("bother.json");
-    
-    // test basic type writing
-    push(JW,"BasicTypes");
-    write(JW,std::string("i16"),i16);
-    write(JW,"u16",u16);
-    write(JW,"i32",i32);
-    write(JW,"u32",u32);
-    write(JW,"i64",i64);
-    write(JW,"u64",u64);
-    write(JW,"f",f);
-    write(JW,"d",d);
-    write(JW,"b",b);
-    pop(JW);
-    
-    // test serializable class writing
-    myclass obj(1234); // non-trivial constructor
-    std::cout << "-- serialisable class writing to 'bother.json'..." << std::endl;
-    write(JW,"obj",obj);
-    JW.write("obj2", obj);
-    
-    std::cout << obj << std::endl;
-    
-    std::vector<myclass> vec;
-    vec.push_back(myclass(1234));
-    vec.push_back(myclass(5678));
-    vec.push_back(myclass(3838));
-    write(JW, "objvec", vec);
-    
-  }
-
-  {
-    JSONReader RD("bother.json");
-    myclass jcopy1;
-    std::vector<myclass> jveccopy1;
-    read(RD,"obj",jcopy1);
-    read(RD,"objvec", jveccopy1);
-    std::cout << "Loaded (JSON) -----------------" << std::endl;
-    std::cout << jcopy1 << std::endl << jveccopy1 << std::endl;
-  }
+//  std::cout << ".:::::: Testing JSON classes "<< std::endl;
+//
+//
+//  {
+//    JSONWriter JW("bother.json");
+//    
+//    // test basic type writing
+//    push(JW,"BasicTypes");
+//    write(JW,std::string("i16"),i16);
+//    write(JW,"u16",u16);
+//    write(JW,"i32",i32);
+//    write(JW,"u32",u32);
+//    write(JW,"i64",i64);
+//    write(JW,"u64",u64);
+//    write(JW,"f",f);
+//    write(JW,"d",d);
+//    write(JW,"b",b);
+//    pop(JW);
+//    
+//    // test serializable class writing
+//    myclass obj(1234); // non-trivial constructor
+//    std::cout << "-- serialisable class writing to 'bother.json'..." << std::endl;
+//    write(JW,"obj",obj);
+//    JW.write("obj2", obj);
+//    
+//    std::cout << obj << std::endl;
+//    
+//    std::vector<myclass> vec;
+//    vec.push_back(myclass(1234));
+//    vec.push_back(myclass(5678));
+//    vec.push_back(myclass(3838));
+//    write(JW, "objvec", vec);
+//    
+//  }
+//
+//  {
+//    JSONReader RD("bother.json");
+//    myclass jcopy1;
+//    std::vector<myclass> jveccopy1;
+//    read(RD,"obj",jcopy1);
+//    read(RD,"objvec", jveccopy1);
+//    std::cout << "Loaded (JSON) -----------------" << std::endl;
+//    std::cout << jcopy1 << std::endl << jveccopy1 << std::endl;
+//  }
   
 /* 
   // This is still work in progress

From b9104f307241558b2d8ec47ee9814a162da5bda8 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 21 Jun 2017 21:08:03 +0100
Subject: [PATCH 003/115] Block CG

---
 lib/algorithms/Algorithms.h | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/lib/algorithms/Algorithms.h b/lib/algorithms/Algorithms.h
index 5123c7a1..3b62c753 100644
--- a/lib/algorithms/Algorithms.h
+++ b/lib/algorithms/Algorithms.h
@@ -44,30 +44,15 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/algorithms/iterative/SchurRedBlack.h>
 #include <Grid/algorithms/iterative/ConjugateGradientMultiShift.h>
 #include <Grid/algorithms/iterative/ConjugateGradientMixedPrec.h>
-
-// Lanczos support
-//#include <Grid/algorithms/iterative/MatrixUtils.h>
+#include <Grid/algorithms/iterative/BlockConjugateGradient.h>
 #include <Grid/algorithms/iterative/ImplicitlyRestartedLanczos.h>
 #include <Grid/algorithms/CoarsenedMatrix.h>
 #include <Grid/algorithms/FFT.h>
 
-// Eigen/lanczos
 // EigCg
-// MCR
 // Pcg
-// Multishift CG
 // Hdcg
 // GCR
 // etc..
 
-// integrator/Leapfrog
-// integrator/Omelyan
-// integrator/ForceGradient
-
-// montecarlo/hmc
-// montecarlo/rhmc
-// montecarlo/metropolis
-// etc...
-
-
 #endif

From 1d7aa673a4cebfde47afb7da8ec9b10a3f681b28 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 21 Jun 2017 21:08:53 +0100
Subject: [PATCH 004/115] Include BlockCG by default

---
 tests/solver/Test_staggered_block_cg_unprec.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/solver/Test_staggered_block_cg_unprec.cc b/tests/solver/Test_staggered_block_cg_unprec.cc
index 8db41e98..5d449f73 100644
--- a/tests/solver/Test_staggered_block_cg_unprec.cc
+++ b/tests/solver/Test_staggered_block_cg_unprec.cc
@@ -27,7 +27,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 #include <Grid/Grid.h>
-#include <Grid/algorithms/iterative/BlockConjugateGradient.h>
 
 using namespace std;
 using namespace Grid;

From 6ebf9f15b7de6b074feb74af0c5037da0fd0730d Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 22 Jun 2017 08:14:34 +0100
Subject: [PATCH 005/115] Splitting communicators first cut

---
 benchmarks/Benchmark_staggered.cc             |  2 +-
 benchmarks/Benchmark_wilson.cc                |  2 +-
 benchmarks/Benchmark_wilson_sweep.cc          |  2 +-
 lib/cartesian/Cartesian_base.h                |  2 +
 lib/cartesian/Cartesian_full.h                | 23 +++++-
 lib/cartesian/Cartesian_red_black.h           | 45 +++++++++--
 lib/communicator/Communicator_base.cc         |  2 +-
 lib/communicator/Communicator_base.h          | 18 ++++-
 lib/communicator/Communicator_mpi.cc          | 76 ++++++++++++++++---
 lib/communicator/Communicator_mpi3.cc         |  9 +++
 lib/communicator/Communicator_none.cc         |  3 +
 lib/communicator/Communicator_shmem.cc        |  5 ++
 lib/qcd/utils/SpaceTimeGrid.cc                | 45 ++++++-----
 lib/util/Lexicographic.h                      |  4 +-
 tests/Test_stencil.cc                         |  2 +-
 tests/core/Test_cshift_red_black.cc           |  2 +-
 tests/core/Test_cshift_red_black_rotate.cc    |  2 +-
 tests/core/Test_fft.cc                        |  2 +-
 tests/core/Test_gpwilson_even_odd.cc          |  2 +-
 tests/core/Test_main.cc                       |  2 +-
 tests/core/Test_staggered.cc                  |  2 +-
 tests/core/Test_wilson_even_odd.cc            |  2 +-
 .../core/Test_wilson_twisted_mass_even_odd.cc |  2 +-
 tests/forces/Test_gp_rect_force.cc            |  2 +-
 tests/forces/Test_laplacian_force.cc          |  2 +-
 tests/forces/Test_rect_force.cc               |  2 +-
 tests/forces/Test_wilson_force.cc             |  2 +-
 tests/smearing/Test_WilsonFlow.cc             |  4 +-
 tests/solver/Test_laplacian.cc                |  2 +-
 tests/solver/Test_staggered_cg_unprec.cc      |  2 +-
 tests/solver/Test_wilson_cg_prec.cc           |  2 +-
 tests/solver/Test_wilson_cg_schur.cc          |  2 +-
 tests/solver/Test_wilson_cg_unprec.cc         |  2 +-
 tests/solver/Test_wilson_cr_unprec.cc         |  2 +-
 34 files changed, 215 insertions(+), 65 deletions(-)

diff --git a/benchmarks/Benchmark_staggered.cc b/benchmarks/Benchmark_staggered.cc
index dc2dcf91..f5325b28 100644
--- a/benchmarks/Benchmark_staggered.cc
+++ b/benchmarks/Benchmark_staggered.cc
@@ -40,7 +40,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/benchmarks/Benchmark_wilson.cc b/benchmarks/Benchmark_wilson.cc
index 671e7654..55042d6a 100644
--- a/benchmarks/Benchmark_wilson.cc
+++ b/benchmarks/Benchmark_wilson.cc
@@ -58,7 +58,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/benchmarks/Benchmark_wilson_sweep.cc b/benchmarks/Benchmark_wilson_sweep.cc
index aae34f6c..a189ac58 100644
--- a/benchmarks/Benchmark_wilson_sweep.cc
+++ b/benchmarks/Benchmark_wilson_sweep.cc
@@ -93,7 +93,7 @@ int main (int argc, char ** argv)
 	  std::cout << latt_size.back() << "\t\t";
 
 	  GridCartesian           Grid(latt_size,simd_layout,mpi_layout);
-	  GridRedBlackCartesian RBGrid(latt_size,simd_layout,mpi_layout);
+	  GridRedBlackCartesian RBGrid(&Grid);
 
 	  GridParallelRNG  pRNG(&Grid); pRNG.SeedFixedIntegers(seeds);
 	  LatticeGaugeField Umu(&Grid); random(pRNG,Umu);
diff --git a/lib/cartesian/Cartesian_base.h b/lib/cartesian/Cartesian_base.h
index 0db6ce0d..25041d17 100644
--- a/lib/cartesian/Cartesian_base.h
+++ b/lib/cartesian/Cartesian_base.h
@@ -49,6 +49,8 @@ public:
     template<class object> friend class Lattice;
 
     GridBase(const std::vector<int> & processor_grid) : CartesianCommunicator(processor_grid) {};
+    GridBase(const std::vector<int> & processor_grid,
+	     const CartesianCommunicator &parent) : CartesianCommunicator(processor_grid,parent) {};
 
     // Physics Grid information.
     std::vector<int> _simd_layout;// Which dimensions get relayed out over simd lanes.
diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h
index b0e47fa4..bced0791 100644
--- a/lib/cartesian/Cartesian_full.h
+++ b/lib/cartesian/Cartesian_full.h
@@ -61,10 +61,29 @@ public:
     virtual int CheckerBoardShift(int source_cb,int dim,int shift, int osite){
       return shift;
     }
+    /////////////////////////////////////////////////////////////////////////
+    // Constructor takes a parent grid and possibly subdivides communicator.
+    /////////////////////////////////////////////////////////////////////////
     GridCartesian(const std::vector<int> &dimensions,
 		  const std::vector<int> &simd_layout,
-		  const std::vector<int> &processor_grid
-		  ) : GridBase(processor_grid)
+		  const std::vector<int> &processor_grid,
+		  GridCartesian &parent) : GridBase(processor_grid,parent)
+    {
+      Init(dimensions,simd_layout,processor_grid);
+    }
+    /////////////////////////////////////////////////////////////////////////
+    // Construct from comm world
+    /////////////////////////////////////////////////////////////////////////
+    GridCartesian(const std::vector<int> &dimensions,
+		  const std::vector<int> &simd_layout,
+		  const std::vector<int> &processor_grid) : GridBase(processor_grid)
+    {
+      Init(dimensions,simd_layout,processor_grid);
+    }
+
+    void Init(const std::vector<int> &dimensions,
+	      const std::vector<int> &simd_layout,
+	      const std::vector<int> &processor_grid)
     {
         ///////////////////////
         // Grid information
diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h
index 3037de00..1f831bc4 100644
--- a/lib/cartesian/Cartesian_red_black.h
+++ b/lib/cartesian/Cartesian_red_black.h
@@ -112,24 +112,57 @@ public:
       }
     };
 
-    GridRedBlackCartesian(const GridBase *base) : GridRedBlackCartesian(base->_fdimensions,base->_simd_layout,base->_processors)  {};
+    ////////////////////////////////////////////////////////////
+    // Create Redblack from original grid; require full grid pointer ?
+    ////////////////////////////////////////////////////////////
+    GridRedBlackCartesian(const GridBase *base) : GridBase(base->_processors,*base)
+    {
+      int dims = base->_ndimension;
+      std::vector<int> checker_dim_mask(dims,1);
+      int checker_dim = 0;
+      Init(base->_fdimensions,base->_simd_layout,base->_processors,checker_dim_mask,checker_dim);
+    };
 
-    GridRedBlackCartesian(const std::vector<int> &dimensions,
+    ////////////////////////////////////////////////////////////
+    // Create redblack from original grid, with non-trivial checker dim mask
+    ////////////////////////////////////////////////////////////
+    GridRedBlackCartesian(const GridBase *base,
+			  const std::vector<int> &checker_dim_mask,
+			  int checker_dim
+			  ) :  GridBase(base->_processors,*base) 
+    {
+      Init(base->_fdimensions,base->_simd_layout,base->_processors,checker_dim_mask,checker_dim)  ;
+    }
+#if 0
+    ////////////////////////////////////////////////////////////
+    // Create redblack grid ;; deprecate these. Should not
+    // need direct creation of redblack without a full grid to base on
+    ////////////////////////////////////////////////////////////
+    GridRedBlackCartesian(const GridBase *base,
+			  const std::vector<int> &dimensions,
 			  const std::vector<int> &simd_layout,
 			  const std::vector<int> &processor_grid,
 			  const std::vector<int> &checker_dim_mask,
 			  int checker_dim
-			  ) :  GridBase(processor_grid) 
+			  ) :  GridBase(processor_grid,*base) 
     {
       Init(dimensions,simd_layout,processor_grid,checker_dim_mask,checker_dim);
     }
-    GridRedBlackCartesian(const std::vector<int> &dimensions,
+
+    ////////////////////////////////////////////////////////////
+    // Create redblack grid
+    ////////////////////////////////////////////////////////////
+    GridRedBlackCartesian(const GridBase *base,
+			  const std::vector<int> &dimensions,
 			  const std::vector<int> &simd_layout,
-			  const std::vector<int> &processor_grid) : GridBase(processor_grid) 
+			  const std::vector<int> &processor_grid) : GridBase(processor_grid,*base) 
     {
       std::vector<int> checker_dim_mask(dimensions.size(),1);
-      Init(dimensions,simd_layout,processor_grid,checker_dim_mask,0);
+      int checker_dim = 0;
+      Init(dimensions,simd_layout,processor_grid,checker_dim_mask,checker_dim);
     }
+#endif
+
     void Init(const std::vector<int> &dimensions,
 	      const std::vector<int> &simd_layout,
 	      const std::vector<int> &processor_grid,
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 557fef48..b042646f 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -60,7 +60,7 @@ void CartesianCommunicator::ShmBufferFreeAll(void) {
 /////////////////////////////////
 // Grid information queries
 /////////////////////////////////
-int                      CartesianCommunicator::Dimensions(void)         { return _ndimension; };
+int                      CartesianCommunicator::Dimensions(void)        { return _ndimension; };
 int                      CartesianCommunicator::IsBoss(void)            { return _processor==0; };
 int                      CartesianCommunicator::BossRank(void)          { return 0; };
 int                      CartesianCommunicator::ThisRank(void)          { return _processor; };
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 12a8429f..ecbb2061 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -68,6 +68,7 @@ class CartesianCommunicator {
   static MPI_Comm communicator_world;
          MPI_Comm communicator;
   typedef MPI_Request CommsRequest_t;
+
 #else 
   typedef int CommsRequest_t;
 #endif
@@ -135,11 +136,24 @@ class CartesianCommunicator {
   // Must call in Grid startup
   ////////////////////////////////////////////////
   static void Init(int *argc, char ***argv);
-  
+
   ////////////////////////////////////////////////
-  // Constructor of any given grid
+  // Constructors to sub-divide a parent communicator
+  // and default to comm world
   ////////////////////////////////////////////////
+  CartesianCommunicator(const std::vector<int> &processors,const CartesianCommunicator &parent);
   CartesianCommunicator(const std::vector<int> &pdimensions_in);
+
+ private:
+#if defined (GRID_COMMS_MPI) 
+  //|| defined (GRID_COMMS_MPI3) 
+  ////////////////////////////////////////////////
+  // Private initialise from an MPI communicator
+  // Can use after an MPI_Comm_split, but hidden from user so private
+  ////////////////////////////////////////////////
+  void InitFromMPICommunicator(const std::vector<int> &processors, MPI_Comm communicator_base);
+#endif
+ public:
   
   ////////////////////////////////////////////////////////////////////////////////////////
   // Wraps MPI_Cart routines, or implements equivalent on other impls
diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index bd2a62fb..b24b60bd 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -53,24 +53,80 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   ShmInitGeneric();
 }
 
-CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors) 
+{
+  InitFromMPICommunicator(processors,communicator_world);
+  std::cout << "Passed communicator world to a new communicator" <<std::endl;
+}
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,const CartesianCommunicator &parent) 
 {
   _ndimension = processors.size();
-  std::vector<int> periodic(_ndimension,1);
+  assert(_ndimension = parent._ndimension);
 
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  // split the communicator
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  std::vector<int> ratio(_ndimension);
+  std::vector<int> rcoor(_ndimension);
+  std::vector<int> scoor(_ndimension);
+
+  int Nsubcomm=1;
+  int Nsubrank=1;
+  for(int d=0;d<_ndimension;d++) {
+    ratio[d] = parent._processors[d] / processors[d];
+    rcoor[d] = parent._processor_coor[d] / processors[d];
+    scoor[d] = parent._processor_coor[d] % processors[d];
+    assert(ratio[d] * processors[d] == parent._processors[d]); // must exactly subdivide
+    Nsubcomm *= ratio[d];
+    Nsubrank *= processors[d];
+  }
+
+  int rlex, slex;
+  Lexicographic::IndexFromCoor(rcoor,rlex,ratio);
+  Lexicographic::IndexFromCoor(scoor,slex,processors);
+
+  MPI_Comm comm_split;
+  int ierr= MPI_Comm_split(communicator_world, rlex, slex,&comm_split);
+  assert(ierr==0);
+
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Set up from the new split communicator
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  InitFromMPICommunicator(processors,comm_split);
+
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Declare victory
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  std::cout << "Divided communicator "<< parent._Nprocessors<<" into "
+	    <<Nsubcomm <<" communicators with " << Nsubrank << " ranks"<<std::endl;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////
+// Take an MPI_Comm and self assemble
+//////////////////////////////////////////////////////////////////////////////////////////////////////
+void CartesianCommunicator::InitFromMPICommunicator(const std::vector<int> &processors, MPI_Comm communicator_base)
+{
+  if ( communicator_base != communicator_world ) {
+    std::cout << "Cartesian communicator created with a non-world communicator"<<std::endl;
+  }
+  _ndimension = processors.size();
+  _processor_coor.resize(_ndimension);
+
+  /////////////////////////////////
+  // Count the requested nodes
+  /////////////////////////////////
   _Nprocessors=1;
   _processors = processors;
-  _processor_coor.resize(_ndimension);
-  
-  MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator);
-  MPI_Comm_rank(communicator,&_processor);
-  MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
-
   for(int i=0;i<_ndimension;i++){
     _Nprocessors*=_processors[i];
   }
-  
-  int Size; 
+
+  std::vector<int> periodic(_ndimension,1);
+  MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],1,&communicator);
+  MPI_Comm_rank(communicator,&_processor);
+  MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
+
+  int Size;
   MPI_Comm_size(communicator,&Size);
   
   assert(Size==_Nprocessors);
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 632eb991..6d7f301c 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -371,6 +371,15 @@ void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &c
   assert(lr!=-1);
   Lexicographic::CoorFromIndex(coor,lr,_processors);
 }
+
+//////////////////////////////////
+// Try to subdivide communicator
+//////////////////////////////////
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,CartesianCommunicator &parent) 
+  : CartesianCommunicator(processors) 
+{
+  std::cout << "Attempts to split MPI3 communicators will fail until implemented" <<std::endl;
+}
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 { 
   int ierr;
diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index 5319ab93..a7484782 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -38,6 +38,9 @@ void CartesianCommunicator::Init(int *argc, char *** arv)
   ShmInitGeneric();
 }
 
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,CartesianCommunicator &parent) 
+  : CartesianCommunicator(processors) {}
+
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
   _processors = processors;
diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc
index 3c76c808..49d32697 100644
--- a/lib/communicator/Communicator_shmem.cc
+++ b/lib/communicator/Communicator_shmem.cc
@@ -75,6 +75,11 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   ShmInitGeneric();
 }
 
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,CartesianCommunicator &parent) 
+  : CartesianCommunicator(processors) 
+{
+  std::cout << "Attempts to split SHMEM communicators will fail " <<std::endl;
+}
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 {
   _ndimension = processors.size();
diff --git a/lib/qcd/utils/SpaceTimeGrid.cc b/lib/qcd/utils/SpaceTimeGrid.cc
index 3ada4a3b..cbbe0aee 100644
--- a/lib/qcd/utils/SpaceTimeGrid.cc
+++ b/lib/qcd/utils/SpaceTimeGrid.cc
@@ -68,18 +68,21 @@ GridRedBlackCartesian *SpaceTimeGrid::makeFiveDimRedBlackGrid(int Ls,const GridC
 {
   int N4=FourDimGrid->_ndimension;
   int cbd=1;
-  std::vector<int> latt5(1,Ls);
-  std::vector<int> simd5(1,1);
-  std::vector<int>  mpi5(1,1);
+  //  std::vector<int> latt5(1,Ls);
+  //  std::vector<int> simd5(1,1);
+  //  std::vector<int>  mpi5(1,1);
   std::vector<int>   cb5(1,0);
     
   for(int d=0;d<N4;d++){
-    latt5.push_back(FourDimGrid->_fdimensions[d]);
-    simd5.push_back(FourDimGrid->_simd_layout[d]);
-     mpi5.push_back(FourDimGrid->_processors[d]);
+    //    latt5.push_back(FourDimGrid->_fdimensions[d]);
+    //    simd5.push_back(FourDimGrid->_simd_layout[d]);
+    //     mpi5.push_back(FourDimGrid->_processors[d]);
       cb5.push_back(  1);
-    }
-  return new GridRedBlackCartesian(latt5,simd5,mpi5,cb5,cbd); 
+  }
+  GridCartesian *tmp = makeFiveDimGrid(Ls,FourDimGrid);
+  GridRedBlackCartesian *ret = new GridRedBlackCartesian(tmp,cb5,cbd); 
+  delete tmp;
+  return ret;
 }
 
 
@@ -99,24 +102,30 @@ GridCartesian         *SpaceTimeGrid::makeFiveDimDWFGrid(int Ls,const GridCartes
   }
   return new GridCartesian(latt5,simd5,mpi5); 
 }
-
+///////////////////////////////////////////////////
+// Interface is inefficient and forces the deletion
+// Pass in the non-redblack grid
+///////////////////////////////////////////////////
 GridRedBlackCartesian *SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(int Ls,const GridCartesian *FourDimGrid)
 {
   int N4=FourDimGrid->_ndimension;
-  int nsimd = FourDimGrid->Nsimd();
   int cbd=1;
-  std::vector<int> latt5(1,Ls);
-  std::vector<int> simd5(1,nsimd);
-  std::vector<int>  mpi5(1,1);
   std::vector<int>   cb5(1,0);
+  //  int nsimd = FourDimGrid->Nsimd();
+  //  std::vector<int> latt5(1,Ls);
+  //  std::vector<int> simd5(1,nsimd);
+  //  std::vector<int>  mpi5(1,1);
     
   for(int d=0;d<N4;d++){
-    latt5.push_back(FourDimGrid->_fdimensions[d]);
-    simd5.push_back(1);
-     mpi5.push_back(FourDimGrid->_processors[d]);
+    //    latt5.push_back(FourDimGrid->_fdimensions[d]);
+    //    simd5.push_back(1);
+    //     mpi5.push_back(FourDimGrid->_processors[d]);
       cb5.push_back(1);
-    }
-  return new GridRedBlackCartesian(latt5,simd5,mpi5,cb5,cbd); 
+  }
+  GridCartesian *tmp         = makeFiveDimDWFGrid(Ls,FourDimGrid);
+  GridRedBlackCartesian *ret = new GridRedBlackCartesian(tmp,cb5,cbd); 
+  delete tmp;
+  return ret;
 }
 
 
diff --git a/lib/util/Lexicographic.h b/lib/util/Lexicographic.h
index 2d4e5df5..b922dba5 100644
--- a/lib/util/Lexicographic.h
+++ b/lib/util/Lexicographic.h
@@ -7,7 +7,7 @@ namespace Grid{
   class Lexicographic {
   public:
 
-    static inline void CoorFromIndex (std::vector<int>& coor,int index,std::vector<int> &dims){
+    static inline void CoorFromIndex (std::vector<int>& coor,int index,const std::vector<int> &dims){
       int nd= dims.size();
       coor.resize(nd);
       for(int d=0;d<nd;d++){
@@ -16,7 +16,7 @@ namespace Grid{
       }
     }
 
-    static inline void IndexFromCoor (std::vector<int>& coor,int &index,std::vector<int> &dims){
+    static inline void IndexFromCoor (const std::vector<int>& coor,int &index,const std::vector<int> &dims){
       int nd=dims.size();
       int stride=1;
       index=0;
diff --git a/tests/Test_stencil.cc b/tests/Test_stencil.cc
index d7bc5a6c..be8ddfd8 100644
--- a/tests/Test_stencil.cc
+++ b/tests/Test_stencil.cc
@@ -49,7 +49,7 @@ int main (int argc, char ** argv)
   double volume = latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
     
   GridCartesian Fine(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian rbFine(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian rbFine(&Fine);
   GridParallelRNG       fRNG(&Fine);
 
   //  fRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9});
diff --git a/tests/core/Test_cshift_red_black.cc b/tests/core/Test_cshift_red_black.cc
index f9269709..c7b0c2f1 100644
--- a/tests/core/Test_cshift_red_black.cc
+++ b/tests/core/Test_cshift_red_black.cc
@@ -47,7 +47,7 @@ int main (int argc, char ** argv)
   mask[0]=0;
 
   GridCartesian         Fine  (latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian RBFine(latt_size,simd_layout,mpi_layout,mask,1);
+  GridRedBlackCartesian RBFine(&Fine,mask,1);
 
   GridParallelRNG      FineRNG(&Fine);  FineRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
diff --git a/tests/core/Test_cshift_red_black_rotate.cc b/tests/core/Test_cshift_red_black_rotate.cc
index 3ef1cd21..aa9e6104 100644
--- a/tests/core/Test_cshift_red_black_rotate.cc
+++ b/tests/core/Test_cshift_red_black_rotate.cc
@@ -47,7 +47,7 @@ int main (int argc, char ** argv)
   mask[0]=0;
 
   GridCartesian         Fine  (latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian RBFine(latt_size,simd_layout,mpi_layout,mask,1);
+  GridRedBlackCartesian RBFine(&Fine,mask,1);
 
   GridParallelRNG      FineRNG(&Fine);  FineRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
 
diff --git a/tests/core/Test_fft.cc b/tests/core/Test_fft.cc
index 877683f0..b2336cfa 100644
--- a/tests/core/Test_fft.cc
+++ b/tests/core/Test_fft.cc
@@ -47,7 +47,7 @@ int main (int argc, char ** argv)
     vol = vol * latt_size[d];
   }
   GridCartesian         GRID(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian RBGRID(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian RBGRID(&GRID);
 
   LatticeComplexD     one(&GRID);
   LatticeComplexD      zz(&GRID);
diff --git a/tests/core/Test_gpwilson_even_odd.cc b/tests/core/Test_gpwilson_even_odd.cc
index fc12fe75..2069eb40 100644
--- a/tests/core/Test_gpwilson_even_odd.cc
+++ b/tests/core/Test_gpwilson_even_odd.cc
@@ -40,7 +40,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/tests/core/Test_main.cc b/tests/core/Test_main.cc
index 378f49bd..a5f1bcd5 100644
--- a/tests/core/Test_main.cc
+++ b/tests/core/Test_main.cc
@@ -84,7 +84,7 @@ int main(int argc, char **argv) {
       double volume = latt_size[0] * latt_size[1] * latt_size[2] * latt_size[3];
 
       GridCartesian Fine(latt_size, simd_layout, mpi_layout);
-      GridRedBlackCartesian rbFine(latt_size, simd_layout, mpi_layout);
+      GridRedBlackCartesian rbFine(&Fine);
       GridParallelRNG FineRNG(&Fine);
       GridSerialRNG SerialRNG;
       GridSerialRNG SerialRNG1;
diff --git a/tests/core/Test_staggered.cc b/tests/core/Test_staggered.cc
index 75531c83..f8f7035a 100644
--- a/tests/core/Test_staggered.cc
+++ b/tests/core/Test_staggered.cc
@@ -40,7 +40,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/tests/core/Test_wilson_even_odd.cc b/tests/core/Test_wilson_even_odd.cc
index a2773244..909a2509 100644
--- a/tests/core/Test_wilson_even_odd.cc
+++ b/tests/core/Test_wilson_even_odd.cc
@@ -51,7 +51,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/tests/core/Test_wilson_twisted_mass_even_odd.cc b/tests/core/Test_wilson_twisted_mass_even_odd.cc
index 4933c36e..adc5c9c9 100644
--- a/tests/core/Test_wilson_twisted_mass_even_odd.cc
+++ b/tests/core/Test_wilson_twisted_mass_even_odd.cc
@@ -52,7 +52,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/tests/forces/Test_gp_rect_force.cc b/tests/forces/Test_gp_rect_force.cc
index bf308749..bb35c77a 100644
--- a/tests/forces/Test_gp_rect_force.cc
+++ b/tests/forces/Test_gp_rect_force.cc
@@ -42,7 +42,7 @@ int main (int argc, char ** argv)
   std::vector<int> mpi_layout  = GridDefaultMpi();
 
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/tests/forces/Test_laplacian_force.cc b/tests/forces/Test_laplacian_force.cc
index c5de1bbc..bae80c0b 100644
--- a/tests/forces/Test_laplacian_force.cc
+++ b/tests/forces/Test_laplacian_force.cc
@@ -42,7 +42,7 @@ int main (int argc, char ** argv)
   std::vector<int> mpi_layout  = GridDefaultMpi();
 
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/tests/forces/Test_rect_force.cc b/tests/forces/Test_rect_force.cc
index c312abeb..a846af30 100644
--- a/tests/forces/Test_rect_force.cc
+++ b/tests/forces/Test_rect_force.cc
@@ -42,7 +42,7 @@ int main (int argc, char ** argv)
   std::vector<int> mpi_layout  = GridDefaultMpi();
 
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/tests/forces/Test_wilson_force.cc b/tests/forces/Test_wilson_force.cc
index 11bac1ae..1f34a48a 100644
--- a/tests/forces/Test_wilson_force.cc
+++ b/tests/forces/Test_wilson_force.cc
@@ -42,7 +42,7 @@ int main (int argc, char ** argv)
   std::vector<int> mpi_layout  = GridDefaultMpi();
 
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   int threads = GridThread::GetThreads();
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
diff --git a/tests/smearing/Test_WilsonFlow.cc b/tests/smearing/Test_WilsonFlow.cc
index 5db00d5d..9d706af0 100644
--- a/tests/smearing/Test_WilsonFlow.cc
+++ b/tests/smearing/Test_WilsonFlow.cc
@@ -71,7 +71,7 @@ int main(int argc, char **argv) {
   std::vector<int> simd_layout = GridDefaultSimd(Nd, vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size, simd_layout, mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size, simd_layout, mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   std::vector<int> seeds({1, 2, 3, 4, 5});
   GridSerialRNG sRNG;
@@ -149,4 +149,4 @@ JSON
 }
 
 
-*/
\ No newline at end of file
+*/
diff --git a/tests/solver/Test_laplacian.cc b/tests/solver/Test_laplacian.cc
index ce4307ab..f163fe54 100644
--- a/tests/solver/Test_laplacian.cc
+++ b/tests/solver/Test_laplacian.cc
@@ -40,7 +40,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   std::vector<int> seeds({1,2,3,4,5});
   GridParallelRNG          pRNG(&Grid);  pRNG.SeedFixedIntegers(seeds);
diff --git a/tests/solver/Test_staggered_cg_unprec.cc b/tests/solver/Test_staggered_cg_unprec.cc
index 5e0358d7..eb33c004 100644
--- a/tests/solver/Test_staggered_cg_unprec.cc
+++ b/tests/solver/Test_staggered_cg_unprec.cc
@@ -57,7 +57,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   std::vector<int> seeds({1,2,3,4});
   GridParallelRNG          pRNG(&Grid);  pRNG.SeedFixedIntegers(seeds);
diff --git a/tests/solver/Test_wilson_cg_prec.cc b/tests/solver/Test_wilson_cg_prec.cc
index 011bc70b..99ddfceb 100644
--- a/tests/solver/Test_wilson_cg_prec.cc
+++ b/tests/solver/Test_wilson_cg_prec.cc
@@ -52,7 +52,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   std::vector<int> seeds({1,2,3,4});
   GridParallelRNG          pRNG(&Grid);  pRNG.SeedFixedIntegers(seeds);
diff --git a/tests/solver/Test_wilson_cg_schur.cc b/tests/solver/Test_wilson_cg_schur.cc
index 7bbf74d3..13ac0090 100644
--- a/tests/solver/Test_wilson_cg_schur.cc
+++ b/tests/solver/Test_wilson_cg_schur.cc
@@ -52,7 +52,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   std::vector<int> seeds({1,2,3,4});
   GridParallelRNG          pRNG(&Grid);  pRNG.SeedFixedIntegers(seeds);
diff --git a/tests/solver/Test_wilson_cg_unprec.cc b/tests/solver/Test_wilson_cg_unprec.cc
index 19c5f854..db227ec8 100644
--- a/tests/solver/Test_wilson_cg_unprec.cc
+++ b/tests/solver/Test_wilson_cg_unprec.cc
@@ -52,7 +52,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   std::vector<int> seeds({1,2,3,4});
   GridParallelRNG          pRNG(&Grid);  pRNG.SeedFixedIntegers(seeds);
diff --git a/tests/solver/Test_wilson_cr_unprec.cc b/tests/solver/Test_wilson_cr_unprec.cc
index 4182c04e..eccd7e74 100644
--- a/tests/solver/Test_wilson_cr_unprec.cc
+++ b/tests/solver/Test_wilson_cr_unprec.cc
@@ -52,7 +52,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   std::vector<int> seeds({1,2,3,4});
   GridParallelRNG          pRNG(&Grid);  pRNG.SeedFixedIntegers(seeds);

From 5e4bea8f2022fcb720bd887a754ee8943b3f38cd Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 22 Jun 2017 08:38:54 +0100
Subject: [PATCH 006/115] Benchmark DWF works

---
 lib/communicator/Communicator_mpi.cc | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index b24b60bd..28a270a0 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -86,19 +86,23 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,
   Lexicographic::IndexFromCoor(scoor,slex,processors);
 
   MPI_Comm comm_split;
-  int ierr= MPI_Comm_split(communicator_world, rlex, slex,&comm_split);
-  assert(ierr==0);
+  if ( Nsubcomm > 1 ) { 
+    int ierr= MPI_Comm_split(communicator_world, rlex, slex,&comm_split);
+    assert(ierr==0);
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    // Declare victory
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    std::cout << "Divided communicator "<< parent._Nprocessors<<" into "
+	      <<Nsubcomm <<" communicators with " << Nsubrank << " ranks"<<std::endl;
+  } else {
+    comm_split = communicator_world;
+  }
 
   //////////////////////////////////////////////////////////////////////////////////////////////////////
   // Set up from the new split communicator
   //////////////////////////////////////////////////////////////////////////////////////////////////////
   InitFromMPICommunicator(processors,comm_split);
 
-  //////////////////////////////////////////////////////////////////////////////////////////////////////
-  // Declare victory
-  //////////////////////////////////////////////////////////////////////////////////////////////////////
-  std::cout << "Divided communicator "<< parent._Nprocessors<<" into "
-	    <<Nsubcomm <<" communicators with " << Nsubrank << " ranks"<<std::endl;
 }
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////

From e504260f3d8381376f9de60028b41c6685e0b93c Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 22 Jun 2017 18:53:11 +0100
Subject: [PATCH 007/115] Able to run a test job splitting into multiple MPI
 subdomains.

---
 lib/algorithms/iterative/ConjugateGradient.h |   4 +-
 lib/cartesian/Cartesian_base.h               |   3 -
 lib/cartesian/Cartesian_full.h               |   2 +-
 lib/communicator/Communicator_mpi.cc         |  54 +++++----
 lib/parallelIO/IldgIO.h                      | 115 ++++++++++++++++---
 lib/parallelIO/IldgIOtypes.h                 |   4 +
 lib/parallelIO/MetaData.h                    |   1 +
 lib/qcd/utils/SpaceTimeGrid.cc               |  19 +--
 8 files changed, 138 insertions(+), 64 deletions(-)

diff --git a/lib/algorithms/iterative/ConjugateGradient.h b/lib/algorithms/iterative/ConjugateGradient.h
index ed453161..5c968e04 100644
--- a/lib/algorithms/iterative/ConjugateGradient.h
+++ b/lib/algorithms/iterative/ConjugateGradient.h
@@ -52,8 +52,8 @@ class ConjugateGradient : public OperatorFunction<Field> {
         MaxIterations(maxit),
         ErrorOnNoConverge(err_on_no_conv){};
 
-  void operator()(LinearOperatorBase<Field> &Linop, const Field &src,
-                  Field &psi) {
+  void operator()(LinearOperatorBase<Field> &Linop, const Field &src, Field &psi) {
+
     psi.checkerboard = src.checkerboard;
     conformable(psi, src);
 
diff --git a/lib/cartesian/Cartesian_base.h b/lib/cartesian/Cartesian_base.h
index 25041d17..a7719ec4 100644
--- a/lib/cartesian/Cartesian_base.h
+++ b/lib/cartesian/Cartesian_base.h
@@ -211,9 +211,6 @@ public:
       assert(lidx<lSites());
       Lexicographic::CoorFromIndex(lcoor,lidx,_ldimensions);
     }
-
-
-
     void GlobalCoorToGlobalIndex(const std::vector<int> & gcoor,int & gidx){
       gidx=0;
       int mult=1;
diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h
index bced0791..eb388f84 100644
--- a/lib/cartesian/Cartesian_full.h
+++ b/lib/cartesian/Cartesian_full.h
@@ -67,7 +67,7 @@ public:
     GridCartesian(const std::vector<int> &dimensions,
 		  const std::vector<int> &simd_layout,
 		  const std::vector<int> &processor_grid,
-		  GridCartesian &parent) : GridBase(processor_grid,parent)
+		  const GridCartesian &parent) : GridBase(processor_grid,parent)
     {
       Init(dimensions,simd_layout,processor_grid);
     }
diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index 28a270a0..7879f518 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -56,46 +56,52 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors) 
 {
   InitFromMPICommunicator(processors,communicator_world);
-  std::cout << "Passed communicator world to a new communicator" <<std::endl;
+  //  std::cout << "Passed communicator world to a new communicator" <<communicator<<std::endl;
 }
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,const CartesianCommunicator &parent) 
 {
   _ndimension = processors.size();
   assert(_ndimension = parent._ndimension);
-
+  
   //////////////////////////////////////////////////////////////////////////////////////////////////////
   // split the communicator
   //////////////////////////////////////////////////////////////////////////////////////////////////////
-  std::vector<int> ratio(_ndimension);
-  std::vector<int> rcoor(_ndimension);
-  std::vector<int> scoor(_ndimension);
+  int Nparent;
+  MPI_Comm_size(parent.communicator,&Nparent);
 
-  int Nsubcomm=1;
-  int Nsubrank=1;
+  int childsize=1;
   for(int d=0;d<_ndimension;d++) {
-    ratio[d] = parent._processors[d] / processors[d];
-    rcoor[d] = parent._processor_coor[d] / processors[d];
-    scoor[d] = parent._processor_coor[d] % processors[d];
-    assert(ratio[d] * processors[d] == parent._processors[d]); // must exactly subdivide
-    Nsubcomm *= ratio[d];
-    Nsubrank *= processors[d];
+    childsize *= processors[d];
   }
+  int Nchild = Nparent/childsize;
+  assert (childsize * Nchild == Nparent);
 
-  int rlex, slex;
-  Lexicographic::IndexFromCoor(rcoor,rlex,ratio);
-  Lexicographic::IndexFromCoor(scoor,slex,processors);
+  int prank;  MPI_Comm_rank(parent.communicator,&prank);
+  int crank = prank % childsize;
+  int ccomm = prank / childsize;
 
   MPI_Comm comm_split;
-  if ( Nsubcomm > 1 ) { 
-    int ierr= MPI_Comm_split(communicator_world, rlex, slex,&comm_split);
+  if ( Nchild > 1 ) { 
+
+    std::cout << GridLogMessage<<"Child communicator of "<< parent.communicator<<std::endl;
+    std::cout << GridLogMessage<<" parent grid    ";
+    for(int d=0;d<parent._processors.size();d++)  std::cout << parent._processors[d] << " ";
+    std::cout<<std::endl;
+
+    std::cout << GridLogMessage<<" child grid    ";
+    for(int d=0;d<processors.size();d++)  std::cout << processors[d] << " ";
+    std::cout<<std::endl;
+
+    int ierr= MPI_Comm_split(parent.communicator, ccomm,crank,&comm_split);
     assert(ierr==0);
     //////////////////////////////////////////////////////////////////////////////////////////////////////
     // Declare victory
     //////////////////////////////////////////////////////////////////////////////////////////////////////
-    std::cout << "Divided communicator "<< parent._Nprocessors<<" into "
-	      <<Nsubcomm <<" communicators with " << Nsubrank << " ranks"<<std::endl;
+    std::cout << GridLogMessage<<"Divided communicator "<< parent._Nprocessors<<" into "
+	      <<Nchild <<" communicators with " << childsize << " ranks"<<std::endl;
   } else {
-    comm_split = communicator_world;
+    comm_split=parent.communicator;
+    //    std::cout << "Passed parental communicator to a new communicator" <<std::endl;
   }
 
   //////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -110,9 +116,9 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,
 //////////////////////////////////////////////////////////////////////////////////////////////////////
 void CartesianCommunicator::InitFromMPICommunicator(const std::vector<int> &processors, MPI_Comm communicator_base)
 {
-  if ( communicator_base != communicator_world ) {
-    std::cout << "Cartesian communicator created with a non-world communicator"<<std::endl;
-  }
+  //  if ( communicator_base != communicator_world ) {
+  //    std::cout << "Cartesian communicator created with a non-world communicator"<<std::endl;
+  //  }
   _ndimension = processors.size();
   _processor_coor.resize(_ndimension);
 
diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h
index 17ce4a06..542f80ef 100644
--- a/lib/parallelIO/IldgIO.h
+++ b/lib/parallelIO/IldgIO.h
@@ -84,10 +84,6 @@ namespace QCD {
    stream << "GRID_";
    stream << ScidacWordMnemonic<stype>();
 
-   //   std::cout << " Lorentz N/S/V/M : " << _LorentzN<<" "<<_LorentzScalar<<"/"<<_LorentzVector<<"/"<<_LorentzMatrix<<std::endl;
-   //   std::cout << " Spin    N/S/V/M : " << _SpinN   <<" "<<_SpinScalar   <<"/"<<_SpinVector   <<"/"<<_SpinMatrix<<std::endl;
-   //   std::cout << " Colour  N/S/V/M : " << _ColourN <<" "<<_ColourScalar <<"/"<<_ColourVector <<"/"<<_ColourMatrix<<std::endl;
-
    if ( _LorentzVector )   stream << "_LorentzVector"<<_LorentzN;
    if ( _LorentzMatrix )   stream << "_LorentzMatrix"<<_LorentzN;
 
@@ -210,19 +206,33 @@ class GridLimeReader : public BinaryIO {
 
     while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { 
 
-      std::cout << GridLogMessage << limeReaderType(LimeR) <<std::endl;
-	
-      if ( strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) )  ) {
+      uint64_t file_bytes =limeReaderBytes(LimeR);
 
+      //      std::cout << GridLogMessage << limeReaderType(LimeR) << " "<< file_bytes <<" bytes "<<std::endl;
+      //      std::cout << GridLogMessage<< " readLimeObject seeking "<<  record_name <<" found record :" <<limeReaderType(LimeR) <<std::endl;
+
+      if ( !strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) )  ) {
+
+	//	std::cout << GridLogMessage<< " readLimeLatticeBinaryObject matches ! " <<std::endl;
+
+	uint64_t PayloadSize = sizeof(sobj) * field._grid->_gsites;
+
+	//	std::cout << "R sizeof(sobj)= " <<sizeof(sobj)<<std::endl;
+	//	std::cout << "R Gsites " <<field._grid->_gsites<<std::endl;
+	//	std::cout << "R Payload expected " <<PayloadSize<<std::endl;
+	//	std::cout << "R file size " <<file_bytes <<std::endl;
+
+	assert(PayloadSize == file_bytes);// Must match or user error
 
 	off_t offset= ftell(File);
+	//	std::cout << " ReadLatticeObject from offset "<<offset << std::endl;
 	BinarySimpleMunger<sobj,sobj> munge;
-	BinaryIO::readLatticeObject< sobj, sobj >(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb);
+	BinaryIO::readLatticeObject< vobj, sobj >(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb);
 
 	/////////////////////////////////////////////
 	// Insist checksum is next record
 	/////////////////////////////////////////////
-	readLimeObject(scidacChecksum_,std::string("scidacChecksum"),record_name);
+	readLimeObject(scidacChecksum_,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM));
 
 	/////////////////////////////////////////////
 	// Verify checksums
@@ -242,9 +252,14 @@ class GridLimeReader : public BinaryIO {
     // should this be a do while; can we miss a first record??
     while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { 
 
+      //      std::cout << GridLogMessage<< " readLimeObject seeking "<< record_name <<" found record :" <<limeReaderType(LimeR) <<std::endl;
+
       uint64_t nbytes = limeReaderBytes(LimeR);//size of this record (configuration)
 
-      if ( strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) )  ) {
+      if ( !strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) )  ) {
+
+	//	std::cout << GridLogMessage<< " readLimeObject matches ! " <<std::endl;
+
 	std::vector<char> xmlc(nbytes+1,'\0');
 	limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR);    
 	XmlReader RD(&xmlc[0],"");
@@ -302,14 +317,18 @@ class GridLimeWriter : public BinaryIO {
       write(WR,object_name,object);
       xmlstring = WR.XmlString();
     }
+    //    std::cout << "WriteLimeObject" << record_name <<std::endl;
     uint64_t nbytes = xmlstring.size();
+    //    std::cout << " xmlstring "<< nbytes<< " " << xmlstring <<std::endl;
     int err;
-    LimeRecordHeader *h = limeCreateHeader(MB, ME,(char *)record_name.c_str(), nbytes); assert(h!= NULL);
+    LimeRecordHeader *h = limeCreateHeader(MB, ME,const_cast<char *>(record_name.c_str()), nbytes); 
+    assert(h!= NULL);
 
     err=limeWriteRecordHeader(h, LimeW);                    assert(err>=0);
     err=limeWriteRecordData(&xmlstring[0], &nbytes, LimeW); assert(err>=0);
     err=limeWriterCloseRecord(LimeW);                       assert(err>=0);
     limeDestroyHeader(h);
+    //    std::cout << " File offset is now"<<ftell(File) << std::endl;
   }
   ////////////////////////////////////////////
   // Write a generic lattice field and csum
@@ -326,6 +345,11 @@ class GridLimeWriter : public BinaryIO {
     uint64_t PayloadSize = sizeof(sobj) * field._grid->_gsites;
     createLimeRecordHeader(record_name, 0, 0, PayloadSize);
 
+
+    //    std::cout << "W sizeof(sobj)"      <<sizeof(sobj)<<std::endl;
+    //    std::cout << "W Gsites "           <<field._grid->_gsites<<std::endl;
+    //    std::cout << "W Payload expected " <<PayloadSize<<std::endl;
+
     ////////////////////////////////////////////////////////////////////
     // NB: FILE and iostream are jointly writing disjoint sequences in the
     // the same file through different file handles (integer units).
@@ -340,6 +364,7 @@ class GridLimeWriter : public BinaryIO {
     //  v) Continue writing scidac record.
     ////////////////////////////////////////////////////////////////////
     off_t offset = ftell(File);
+    //    std::cout << " Writing to offset "<<offset << std::endl;
     std::string format = getFormatString<vobj>();
     BinarySimpleMunger<sobj,sobj> munge;
     BinaryIO::writeLatticeObject<vobj,sobj>(field, filename, munge, offset, format,nersc_csum,scidac_csuma,scidac_csumb);
@@ -354,7 +379,7 @@ class GridLimeWriter : public BinaryIO {
     checksum.suma= streama.str();
     checksum.sumb= streamb.str();
     std::cout << GridLogMessage<<" writing scidac checksums "<<std::hex<<scidac_csuma<<"/"<<scidac_csumb<<std::dec<<std::endl;
-    writeLimeObject(0,1,checksum,std::string("scidacChecksum"    ),std::string(SCIDAC_CHECKSUM));
+    writeLimeObject(0,1,checksum,std::string("scidacChecksum"),std::string(SCIDAC_CHECKSUM));
   }
 };
 
@@ -371,11 +396,9 @@ class ScidacWriter : public GridLimeWriter {
   ////////////////////////////////////////////////
   // Write generic lattice field in scidac format
   ////////////////////////////////////////////////
-   template <class vobj, class userRecord>
+  template <class vobj, class userRecord>
   void writeScidacFieldRecord(Lattice<vobj> &field,userRecord _userRecord) 
   {
-    typedef typename vobj::scalar_object sobj;
-    uint64_t nbytes;
     GridBase * grid = field._grid;
 
     ////////////////////////////////////////
@@ -397,6 +420,66 @@ class ScidacWriter : public GridLimeWriter {
   }
 };
 
+
+class ScidacReader : public GridLimeReader {
+ public:
+
+   template<class SerialisableUserFile>
+   void readScidacFileRecord(GridBase *grid,SerialisableUserFile &_userFile)
+   {
+     scidacFile    _scidacFile(grid);
+     readLimeObject(_scidacFile,_scidacFile.SerialisableClassName(),std::string(SCIDAC_PRIVATE_FILE_XML));
+     readLimeObject(_userFile,_userFile.SerialisableClassName(),std::string(SCIDAC_FILE_XML));
+   }
+  ////////////////////////////////////////////////
+  // Write generic lattice field in scidac format
+  ////////////////////////////////////////////////
+  template <class vobj, class userRecord>
+  void readScidacFieldRecord(Lattice<vobj> &field,userRecord &_userRecord) 
+  {
+    typedef typename vobj::scalar_object sobj;
+    GridBase * grid = field._grid;
+
+    ////////////////////////////////////////
+    // fill the Grid header
+    ////////////////////////////////////////
+    FieldMetaData header;
+    scidacRecord  _scidacRecord;
+    scidacFile    _scidacFile;
+
+    //////////////////////////////////////////////
+    // Fill the Lime file record by record
+    //////////////////////////////////////////////
+    readLimeObject(header ,std::string("FieldMetaData"),std::string(GRID_FORMAT)); // Open message 
+    readLimeObject(_userRecord,_userRecord.SerialisableClassName(),std::string(SCIDAC_RECORD_XML));
+    readLimeObject(_scidacRecord,_scidacRecord.SerialisableClassName(),std::string(SCIDAC_PRIVATE_RECORD_XML));
+    readLimeLatticeBinaryObject(field,std::string(ILDG_BINARY_DATA));
+  }
+  void skipPastBinaryRecord(void) {
+    std::string rec_name(ILDG_BINARY_DATA);
+    while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { 
+      if ( !strncmp(limeReaderType(LimeR), rec_name.c_str(),strlen(rec_name.c_str()) )  ) {
+	skipPastObjectRecord(std::string(SCIDAC_CHECKSUM));
+	return;
+      }
+    }    
+  }
+  void skipPastObjectRecord(std::string rec_name) {
+    while ( limeReaderNextRecord(LimeR) == LIME_SUCCESS ) { 
+      if ( !strncmp(limeReaderType(LimeR), rec_name.c_str(),strlen(rec_name.c_str()) )  ) {
+	return;
+      }
+    }
+  }
+  void skipScidacFieldRecord() {
+    skipPastObjectRecord(std::string(GRID_FORMAT));
+    skipPastObjectRecord(std::string(SCIDAC_RECORD_XML));
+    skipPastObjectRecord(std::string(SCIDAC_PRIVATE_RECORD_XML));
+    skipPastBinaryRecord();
+  }
+};
+
+
 class IldgWriter : public ScidacWriter {
  public:
 
@@ -425,8 +508,6 @@ class IldgWriter : public ScidacWriter {
     typedef iLorentzColourMatrix<vsimd> vobj;
     typedef typename vobj::scalar_object sobj;
 
-    uint64_t nbytes;
-
     ////////////////////////////////////////
     // fill the Grid header
     ////////////////////////////////////////
diff --git a/lib/parallelIO/IldgIOtypes.h b/lib/parallelIO/IldgIOtypes.h
index c3a5321c..53664b49 100644
--- a/lib/parallelIO/IldgIOtypes.h
+++ b/lib/parallelIO/IldgIOtypes.h
@@ -64,6 +64,10 @@ namespace Grid {
 // file compatability, so should be correct to assume the undocumented but defacto file structure.
 /////////////////////////////////////////////////////////////////////////////////
 
+struct emptyUserRecord : Serializable { 
+  GRID_SERIALIZABLE_CLASS_MEMBERS(emptyUserRecord,int,dummy);
+};
+
 ////////////////////////
 // Scidac private file xml
 // <?xml version="1.0" encoding="UTF-8"?><scidacFile><version>1.1</version><spacetime>4</spacetime><dims>16 16 16 32 </dims><volfmt>0</volfmt></scidacFile>
diff --git a/lib/parallelIO/MetaData.h b/lib/parallelIO/MetaData.h
index 6d45d0a5..8fc0d777 100644
--- a/lib/parallelIO/MetaData.h
+++ b/lib/parallelIO/MetaData.h
@@ -104,6 +104,7 @@ namespace Grid {
       header.nd = nd;
       header.dimension.resize(nd);
       header.boundary.resize(nd);
+      header.data_start = 0;
       for(int d=0;d<nd;d++) {
 	header.dimension[d] = grid->_fdimensions[d];
       }
diff --git a/lib/qcd/utils/SpaceTimeGrid.cc b/lib/qcd/utils/SpaceTimeGrid.cc
index cbbe0aee..b2b5d9c8 100644
--- a/lib/qcd/utils/SpaceTimeGrid.cc
+++ b/lib/qcd/utils/SpaceTimeGrid.cc
@@ -60,7 +60,7 @@ GridCartesian         *SpaceTimeGrid::makeFiveDimGrid(int Ls,const GridCartesian
     simd5.push_back(FourDimGrid->_simd_layout[d]);
      mpi5.push_back(FourDimGrid->_processors[d]);
   }
-  return new GridCartesian(latt5,simd5,mpi5); 
+  return new GridCartesian(latt5,simd5,mpi5,*FourDimGrid); 
 }
 
 
@@ -68,15 +68,8 @@ GridRedBlackCartesian *SpaceTimeGrid::makeFiveDimRedBlackGrid(int Ls,const GridC
 {
   int N4=FourDimGrid->_ndimension;
   int cbd=1;
-  //  std::vector<int> latt5(1,Ls);
-  //  std::vector<int> simd5(1,1);
-  //  std::vector<int>  mpi5(1,1);
   std::vector<int>   cb5(1,0);
-    
   for(int d=0;d<N4;d++){
-    //    latt5.push_back(FourDimGrid->_fdimensions[d]);
-    //    simd5.push_back(FourDimGrid->_simd_layout[d]);
-    //     mpi5.push_back(FourDimGrid->_processors[d]);
       cb5.push_back(  1);
   }
   GridCartesian *tmp = makeFiveDimGrid(Ls,FourDimGrid);
@@ -100,7 +93,7 @@ GridCartesian         *SpaceTimeGrid::makeFiveDimDWFGrid(int Ls,const GridCartes
     simd5.push_back(1);
      mpi5.push_back(FourDimGrid->_processors[d]);
   }
-  return new GridCartesian(latt5,simd5,mpi5); 
+  return new GridCartesian(latt5,simd5,mpi5,*FourDimGrid); 
 }
 ///////////////////////////////////////////////////
 // Interface is inefficient and forces the deletion
@@ -111,15 +104,7 @@ GridRedBlackCartesian *SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(int Ls,const Gr
   int N4=FourDimGrid->_ndimension;
   int cbd=1;
   std::vector<int>   cb5(1,0);
-  //  int nsimd = FourDimGrid->Nsimd();
-  //  std::vector<int> latt5(1,Ls);
-  //  std::vector<int> simd5(1,nsimd);
-  //  std::vector<int>  mpi5(1,1);
-    
   for(int d=0;d<N4;d++){
-    //    latt5.push_back(FourDimGrid->_fdimensions[d]);
-    //    simd5.push_back(1);
-    //     mpi5.push_back(FourDimGrid->_processors[d]);
       cb5.push_back(1);
   }
   GridCartesian *tmp         = makeFiveDimDWFGrid(Ls,FourDimGrid);

From 600d7ddc2ea54c956a63d9cf0de448909130f51e Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 22 Jun 2017 18:54:34 +0100
Subject: [PATCH 008/115] Proof of concept : Multi RHS solver, running
 independent solves on different ranks

---
 tests/solver/Test_dwf_mrhs_cg.cc | 175 +++++++++++++++++++++++++++++++
 1 file changed, 175 insertions(+)
 create mode 100644 tests/solver/Test_dwf_mrhs_cg.cc

diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc
new file mode 100644
index 00000000..fa940390
--- /dev/null
+++ b/tests/solver/Test_dwf_mrhs_cg.cc
@@ -0,0 +1,175 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_dwf_mrhs_cg.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+#include <Grid/algorithms/iterative/BlockConjugateGradient.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+int main (int argc, char ** argv)
+{
+  typedef typename DomainWallFermionR::FermionField FermionField; 
+  typedef typename DomainWallFermionR::ComplexField ComplexField; 
+  typename DomainWallFermionR::ImplParams params; 
+
+  const int Ls=8;
+
+  Grid_init(&argc,&argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+  std::vector<int> mpi_split (mpi_layout.size(),1);
+
+  std::cout << "UGrid (world root)"<<std::endl;
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+
+  std::cout << "FGrid (child of UGrid)"<<std::endl;
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+
+  int nrhs = UGrid->RankCount() ;
+
+  /////////////////////////////////////////////
+  // Split into 1^4 mpi communicators
+  /////////////////////////////////////////////
+  std::cout << "SGrid (world root)"<<std::endl;
+  GridCartesian         * SGrid = new GridCartesian(GridDefaultLatt(),
+						    GridDefaultSimd(Nd,vComplex::Nsimd()),
+						    mpi_split,
+						    *UGrid); 
+
+  GridCartesian         * SFGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,SGrid);
+  std::cout << "SFGrid"<<std::endl;
+  GridRedBlackCartesian * SrbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(SGrid);
+  std::cout << "SrbGrid"<<std::endl;
+  GridRedBlackCartesian * SFrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,SGrid);
+  std::cout << "SFrbGrid"<<std::endl;
+
+  ///////////////////////////////////////////////
+  // Set up the problem as a 4d spreadout job
+  ///////////////////////////////////////////////
+  std::vector<int> seeds({1,2,3,4});
+
+  GridParallelRNG pRNG(UGrid );  pRNG.SeedFixedIntegers(seeds);
+  GridParallelRNG pRNG5(FGrid);  pRNG5.SeedFixedIntegers(seeds);
+  std::vector<FermionField>    src(nrhs,FGrid);
+  std::vector<FermionField> result(nrhs,FGrid);
+
+  for(int s=0;s<nrhs;s++) random(pRNG5,src[s]);
+  for(int s=0;s<nrhs;s++) result[s] = zero;
+
+  LatticeGaugeField Umu(UGrid); SU3::HotConfiguration(pRNG,Umu);
+
+  ///////////////////////////////////////////////////////////////
+  // Bounce these fields to disk
+  ///////////////////////////////////////////////////////////////
+
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+  std::cout << GridLogMessage << " Writing out in parallel view "<<std::endl;
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+  emptyUserRecord record;
+  std::string file("./scratch.scidac");
+  {
+    ScidacWriter _ScidacWriter;
+    _ScidacWriter.open(file);
+    _ScidacWriter.writeScidacFieldRecord(Umu,record);
+    for(int n=0;n<nrhs;n++){
+      _ScidacWriter.writeScidacFieldRecord(src[n],record);
+      
+    }
+    _ScidacWriter.close();
+  }
+
+
+
+  //////////////////////////////////////////
+  // Read back into single rank fields
+  //////////////////////////////////////////
+
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+  std::cout << GridLogMessage << " Reading back in the single process view "<<std::endl;
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+
+  int me = UGrid->ThisRank();
+  LatticeGaugeField s_Umu(SGrid);
+  FermionField s_src(SFGrid);
+  FermionField s_res(SFGrid);
+  {
+    ScidacReader  _ScidacReader;
+    _ScidacReader.open(file);
+    std::cout << GridLogMessage << " Opened file "<<std::endl;
+    _ScidacReader.readScidacFieldRecord(s_Umu,record);
+    std::cout << GridLogMessage << " Read gauge field "<<std::endl;
+    for(int n=0;n<nrhs;n++){
+      if ( n==me ) { 
+	std::cout << GridLogMessage << " Read record  "<<n<<std::endl;
+	_ScidacReader.readScidacFieldRecord(s_src,record);
+      } else { 
+	std::cout << GridLogMessage << " Skip record  "<<n<<std::endl;
+	_ScidacReader.skipScidacFieldRecord();
+      }      
+    }
+    _ScidacReader.close();
+  }
+
+  ///////////////////////////////////////////////////////////////
+  // Set up N-solvers as trivially parallel
+  ///////////////////////////////////////////////////////////////
+
+  RealD mass=0.01;
+  RealD M5=1.8;
+  DomainWallFermionR Ddwf(s_Umu,*SFGrid,*SFrbGrid,*SGrid,*SrbGrid,mass,M5);
+
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+  std::cout << GridLogMessage << " Calling DWF CG "<<std::endl;
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+
+  MdagMLinearOperator<DomainWallFermionR,FermionField> HermOp(Ddwf);
+  ConjugateGradient<FermionField> CG((1.0e-8/(me+1)),10000);
+  s_res = zero;
+  CG(HermOp,s_src,s_res);
+
+  ///////////////////////////////////////
+  // Share the information
+  ///////////////////////////////////////
+  std::vector<uint32_t> iterations(nrhs,0);
+  iterations[me] = CG.IterationsToComplete;
+
+  for(int n=0;n<nrhs;n++){
+    UGrid->GlobalSum(iterations[n]);
+  }
+
+  /////////////////////////////////////////////////////////////
+  // Report how long they all took
+  /////////////////////////////////////////////////////////////
+  for(int r=0;r<nrhs;r++){
+    std::cout << " Rank "<<r<<" "<< iterations[r]<<" CG iterations"<<std::endl;
+  }
+  Grid_finalize();
+}

From 1feddf4ba65c2a9895ce16ae33e1e443cc67514b Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 22 Jun 2017 19:32:41 +0100
Subject: [PATCH 009/115] const fixes

---
 lib/communicator/Communicator_mpi3.cc  | 2 +-
 lib/communicator/Communicator_none.cc  | 2 +-
 lib/communicator/Communicator_shmem.cc | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 6d7f301c..22445f60 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -375,7 +375,7 @@ void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &c
 //////////////////////////////////
 // Try to subdivide communicator
 //////////////////////////////////
-CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,CartesianCommunicator &parent) 
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,const CartesianCommunicator &parent) 
   : CartesianCommunicator(processors) 
 {
   std::cout << "Attempts to split MPI3 communicators will fail until implemented" <<std::endl;
diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index a7484782..40feefec 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -38,7 +38,7 @@ void CartesianCommunicator::Init(int *argc, char *** arv)
   ShmInitGeneric();
 }
 
-CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,CartesianCommunicator &parent) 
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,const CartesianCommunicator &parent) 
   : CartesianCommunicator(processors) {}
 
 CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
diff --git a/lib/communicator/Communicator_shmem.cc b/lib/communicator/Communicator_shmem.cc
index 49d32697..ed49285d 100644
--- a/lib/communicator/Communicator_shmem.cc
+++ b/lib/communicator/Communicator_shmem.cc
@@ -75,7 +75,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   ShmInitGeneric();
 }
 
-CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,CartesianCommunicator &parent) 
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,const CartesianCommunicator &parent) 
   : CartesianCommunicator(processors) 
 {
   std::cout << "Attempts to split SHMEM communicators will fail " <<std::endl;

From e51475703a404e80a6430d2fd7342dcf53518206 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 23 Jun 2017 09:42:21 +0100
Subject: [PATCH 010/115] Ticking off lots on the TODO list

---
 TODO | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/TODO b/TODO
index 001c6c0c..f0c022ca 100644
--- a/TODO
+++ b/TODO
@@ -2,18 +2,24 @@ TODO:
 ---------------
 
 Large item work list:
-1)- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O
 
+1)- BG/Q port and check
 2)- Christoph's local basis expansion Lanczos
-3)- BG/Q port and check
-4)- Precision conversion and sort out localConvert      <-- partial
-  - Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet
-5)- Physical propagator interface
-6)- Conserved currents
-7)- Multigrid Wilson and DWF, compare to other Multigrid implementations
-8)- HDCR resume
+--
+3a)- RNG I/O in ILDG/SciDAC (minor)
+3b)- Precision conversion and sort out localConvert      <-- partial/easy
+3c)- Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet
+
+4)- Physical propagator interface
+
+5)- Conserved currents
+
+6)- Multigrid Wilson and DWF, compare to other Multigrid implementations
+
+7)- HDCR resume
 
 Recent DONE 
+-- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O ; <-- DONE ; bmark cori
 -- Lanczos Remove DenseVector, DenseMatrix; Use Eigen instead. <-- DONE
 -- GaugeFix into central location                      <-- DONE
 -- Scidac and Ildg metadata handling                   <-- DONE

From 349d75e48379c353c886966cebcb0eeb944bb624 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 23 Jun 2017 02:57:59 -0700
Subject: [PATCH 011/115] Precision fix

---
 lib/lattice/Lattice_reduction.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/lattice/Lattice_reduction.h b/lib/lattice/Lattice_reduction.h
index c5b20f3c..fc1ccadb 100644
--- a/lib/lattice/Lattice_reduction.h
+++ b/lib/lattice/Lattice_reduction.h
@@ -540,7 +540,7 @@ static void sliceInnerProductMatrix(  Eigen::MatrixXcd &mat, const Lattice<vobj>
       for(int i=0;i<Nblock;i++){
       for(int j=0;j<Nblock;j++){
 	auto tmp = innerProduct(Left[i],Right[j]);
-	vector_typeD rtmp = TensorRemove(tmp);
+	auto rtmp = TensorRemove(tmp);
 	mat_thread(i,j) += Reduce(rtmp);
       }}
     }}

From 0165bcb58e5209df6a5dfa9d6429297ce06fa53d Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 23 Jun 2017 23:10:24 +0100
Subject: [PATCH 012/115] Added an update to TODO list

---
 TODO | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/TODO b/TODO
index f0c022ca..a778f476 100644
--- a/TODO
+++ b/TODO
@@ -3,19 +3,17 @@ TODO:
 
 Large item work list:
 
-1)- BG/Q port and check
+1)- BG/Q port and check ; Andrew says ok.
 2)- Christoph's local basis expansion Lanczos
 --
 3a)- RNG I/O in ILDG/SciDAC (minor)
 3b)- Precision conversion and sort out localConvert      <-- partial/easy
 3c)- Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet
-
+--
 4)- Physical propagator interface
-
 5)- Conserved currents
-
+--
 6)- Multigrid Wilson and DWF, compare to other Multigrid implementations
-
 7)- HDCR resume
 
 Recent DONE 

From 869b99ec1efde04d94bdd02eb041a457accb930e Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sat, 24 Jun 2017 10:55:54 +0100
Subject: [PATCH 013/115] Threaded calls to multiple communicators

---
 lib/communicator/Communicator_mpit.cc | 260 ++++++++++++++++++++++++++
 1 file changed, 260 insertions(+)
 create mode 100644 lib/communicator/Communicator_mpit.cc

diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc
new file mode 100644
index 00000000..07522900
--- /dev/null
+++ b/lib/communicator/Communicator_mpit.cc
@@ -0,0 +1,260 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/communicator/Communicator_mpi.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/GridCore.h>
+#include <Grid/GridQCDcore.h>
+#include <Grid/qcd/action/ActionCore.h>
+#include <mpi.h>
+
+namespace Grid {
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Info that is setup once and indept of cartesian layout
+///////////////////////////////////////////////////////////////////////////////////////////////////
+MPI_Comm CartesianCommunicator::communicator_world;
+
+// Should error check all MPI calls.
+void CartesianCommunicator::Init(int *argc, char ***argv) {
+  int flag;
+  int provided;
+  MPI_Initialized(&flag); // needed to coexist with other libs apparently
+  if ( !flag ) {
+    MPI_Init_thread(argc,argv,MPI_THREAD_MULTIPLE,&provided);
+    if ( provided != MPI_THREAD_MULTIPLE ) {
+      QCD::WilsonKernelsStatic::Comms = QCD::WilsonKernelsStatic::CommsThenCompute;
+    }
+  }
+  MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
+  ShmInitGeneric();
+}
+
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
+{
+  _ndimension = processors.size();
+  std::vector<int> periodic(_ndimension,1);
+
+  _Nprocessors=1;
+  _processors = processors;
+  _processor_coor.resize(_ndimension);
+  
+  MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator);
+  MPI_Comm_rank(communicator,&_processor);
+  MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
+
+  for(int i=0;i<_ndimension;i++){
+    _Nprocessors*=_processors[i];
+  }
+
+  communicator_halo.resize (2*_ndimension);
+  for(int i=0;i<_ndimension*2;i++){
+    MPI_Comm_dup(communicator,&communicator_halo[i]);
+  }
+  
+  int Size; 
+  MPI_Comm_size(communicator,&Size);
+  
+  assert(Size==_Nprocessors);
+}
+void CartesianCommunicator::GlobalSum(uint32_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSum(uint64_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalXOR(uint32_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_BXOR,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalXOR(uint64_t &u){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT64_T,MPI_BXOR,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSum(float &f){
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,&f,1,MPI_FLOAT,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSumVector(float *f,int N)
+{
+  int ierr=MPI_Allreduce(MPI_IN_PLACE,f,N,MPI_FLOAT,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSum(double &d)
+{
+  int ierr = MPI_Allreduce(MPI_IN_PLACE,&d,1,MPI_DOUBLE,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::GlobalSumVector(double *d,int N)
+{
+  int ierr = MPI_Allreduce(MPI_IN_PLACE,d,N,MPI_DOUBLE,MPI_SUM,communicator);
+  assert(ierr==0);
+}
+void CartesianCommunicator::ShiftedRanks(int dim,int shift,int &source,int &dest)
+{
+  int ierr=MPI_Cart_shift(communicator,dim,shift,&source,&dest);
+  assert(ierr==0);
+}
+int CartesianCommunicator::RankFromProcessorCoor(std::vector<int> &coor)
+{
+  int rank;
+  int ierr=MPI_Cart_rank  (communicator, &coor[0], &rank);
+  assert(ierr==0);
+  return rank;
+}
+void  CartesianCommunicator::ProcessorCoorFromRank(int rank, std::vector<int> &coor)
+{
+  coor.resize(_ndimension);
+  int ierr=MPI_Cart_coords  (communicator, rank, _ndimension,&coor[0]);
+  assert(ierr==0);
+}
+
+// Basic Halo comms primitive
+void CartesianCommunicator::SendToRecvFrom(void *xmit,
+					   int dest,
+					   void *recv,
+					   int from,
+					   int bytes)
+{
+  std::vector<CommsRequest_t> reqs(0);
+  SendToRecvFromBegin(reqs,xmit,dest,recv,from,bytes);
+  SendToRecvFromComplete(reqs);
+}
+
+void CartesianCommunicator::SendRecvPacket(void *xmit,
+					   void *recv,
+					   int sender,
+					   int receiver,
+					   int bytes)
+{
+  MPI_Status stat;
+  assert(sender != receiver);
+  int tag = sender;
+  if ( _processor == sender ) {
+    MPI_Send(xmit, bytes, MPI_CHAR,receiver,tag,communicator);
+  }
+  if ( _processor == receiver ) { 
+    MPI_Recv(recv, bytes, MPI_CHAR,sender,tag,communicator,&stat);
+  }
+}
+
+// Basic Halo comms primitive
+void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+						void *xmit,
+						int dest,
+						void *recv,
+						int from,
+						int bytes)
+{
+  int myrank = _processor;
+  int ierr;
+  if ( CommunicatorPolicy == CommunicatorPolicyConcurrent ) { 
+    MPI_Request xrq;
+    MPI_Request rrq;
+
+    ierr =MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+    ierr|=MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+    
+    assert(ierr==0);
+    list.push_back(xrq);
+    list.push_back(rrq);
+  } else { 
+    // Give the CPU to MPI immediately; can use threads to overlap optionally
+    ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,dest,myrank,
+		      recv,bytes,MPI_CHAR,from, from,
+		      communicator,MPI_STATUS_IGNORE);
+    assert(ierr==0);
+  }
+}
+void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &list)
+{
+  if ( CommunicatorPolicy == CommunicatorPolicyConcurrent ) { 
+    int nreq=list.size();
+    std::vector<MPI_Status> status(nreq);
+    int ierr = MPI_Waitall(nreq,&list[0],&status[0]);
+    assert(ierr==0);
+  }
+}
+
+void CartesianCommunicator::Barrier(void)
+{
+  int ierr = MPI_Barrier(communicator);
+  assert(ierr==0);
+}
+
+void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
+{
+  int ierr=MPI_Bcast(data,
+		     bytes,
+		     MPI_BYTE,
+		     root,
+		     communicator);
+  assert(ierr==0);
+}
+  ///////////////////////////////////////////////////////
+  // Should only be used prior to Grid Init finished.
+  // Check for this?
+  ///////////////////////////////////////////////////////
+int CartesianCommunicator::RankWorld(void){ 
+  int r; 
+  MPI_Comm_rank(communicator_world,&r);
+  return r;
+}
+void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
+{
+  int ierr= MPI_Bcast(data,
+		      bytes,
+		      MPI_BYTE,
+		      root,
+		      communicator_world);
+  assert(ierr==0);
+}
+
+  double CartesianCommunicator::StencilSendToRecvFromBegin(int dir,
+							   std::vector<CommsRequest_t> &list,
+							   void *xmit,
+							   int xmit_to_rank,
+							   void *recv,
+							   int recv_from_rank,
+							   int bytes)
+{
+  int myrank = _processor;
+  int ierr;
+  // Give the CPU to MPI immediately; can use threads to overlap optionally
+  ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,dest,myrank,
+		    recv,bytes,MPI_CHAR,from, from,
+		    communicator_halo[dir],MPI_STATUS_IGNORE);
+  assert(ierr==0);
+  return 2.0*bytes;
+}
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall){ };
+
+
+
+}
+

From 54e94360ad06cde7edbaeede2cf18eb0d5a1227b Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sat, 24 Jun 2017 23:10:24 +0100
Subject: [PATCH 014/115] Experimental: Multiple communicators to see if we can
 avoid thread locks in --enable-comms=mpit

---
 benchmarks/Benchmark_comms.cc             | 27 ++++++-----
 configure.ac                              | 10 ++---
 lib/Makefile.am                           |  4 +-
 lib/communicator/Communicator_base.cc     | 22 +++++----
 lib/communicator/Communicator_base.h      | 20 +++++----
 lib/communicator/Communicator_mpi3.cc     | 12 ++---
 lib/communicator/Communicator_mpit.cc     | 26 ++++++-----
 lib/cshift/Cshift.h                       |  2 +-
 lib/log/Log.cc                            |  2 +-
 lib/parallelIO/BinaryIO.h                 |  2 +-
 lib/qcd/action/fermion/WilsonFermion5D.cc | 55 +++++++++++------------
 lib/stencil/Stencil.h                     | 45 ++++++++++++++++---
 lib/util/Init.cc                          |  2 +-
 13 files changed, 139 insertions(+), 90 deletions(-)

diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc
index 532532f8..753b8a58 100644
--- a/benchmarks/Benchmark_comms.cc
+++ b/benchmarks/Benchmark_comms.cc
@@ -68,7 +68,7 @@ int main (int argc, char ** argv)
 
   int Nloop=100;
   int nmu=0;
-  int maxlat=24;
+  int maxlat=32;
   for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
 
   std::cout << GridLogMessage << "Number of iterations to average: "<< Nloop << std::endl;
@@ -80,7 +80,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   header();
   for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=32;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
       				    lat*mpi_layout[1],
@@ -163,7 +163,7 @@ int main (int argc, char ** argv)
   header();
 
   for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=32;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
 
       std::vector<int> latt_size  ({lat,lat,lat,lat});
 
@@ -249,7 +249,7 @@ int main (int argc, char ** argv)
   header();
 
   for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=32;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
       				    lat*mpi_layout[1],
@@ -299,7 +299,7 @@ int main (int argc, char ** argv)
 					      xmit_to_rank,
 					      (void *)&rbuf[mu][0],
 					      recv_from_rank,
-					      bytes);
+					      bytes,mu);
 	
 	    comm_proc = mpi_layout[mu]-1;
 	  
@@ -310,11 +310,11 @@ int main (int argc, char ** argv)
 					      xmit_to_rank,
 					      (void *)&rbuf[mu+4][0],
 					      recv_from_rank,
-					      bytes);
+					      bytes,mu+4);
 	  
 	  }
 	}
-	Grid.StencilSendToRecvFromComplete(requests);
+	Grid.StencilSendToRecvFromComplete(requests,0);
 	Grid.Barrier();
 	double stop=usecond();
 	t_time[i] = stop-start; // microseconds
@@ -346,7 +346,7 @@ int main (int argc, char ** argv)
   header();
 
   for(int lat=4;lat<=maxlat;lat+=4){
-    for(int Ls=8;Ls<=32;Ls*=2){
+    for(int Ls=8;Ls<=8;Ls*=2){
 
       std::vector<int> latt_size  ({lat*mpi_layout[0],
       				    lat*mpi_layout[1],
@@ -393,8 +393,8 @@ int main (int argc, char ** argv)
 					      xmit_to_rank,
 					      (void *)&rbuf[mu][0],
 					      recv_from_rank,
-					      bytes);
-	    Grid.StencilSendToRecvFromComplete(requests);
+					      bytes,mu);
+	    Grid.StencilSendToRecvFromComplete(requests,mu);
 	    requests.resize(0);
 
 	    comm_proc = mpi_layout[mu]-1;
@@ -406,8 +406,8 @@ int main (int argc, char ** argv)
 					      xmit_to_rank,
 					      (void *)&rbuf[mu+4][0],
 					      recv_from_rank,
-					      bytes);
-	    Grid.StencilSendToRecvFromComplete(requests);
+					      bytes,mu+4);
+	    Grid.StencilSendToRecvFromComplete(requests,mu+4);
 	    requests.resize(0);
 	  
 	  }
@@ -435,6 +435,9 @@ int main (int argc, char ** argv)
  
     }
   }    
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= All done; Bye Bye"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
 
   Grid_finalize();
 }
diff --git a/configure.ac b/configure.ac
index f7284d48..9a596bd0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -324,14 +324,14 @@ case ${ac_COMMS} in
         AC_DEFINE([GRID_COMMS_NONE],[1],[GRID_COMMS_NONE] )
         comms_type='none'
      ;;
-     mpi3l*)
-       AC_DEFINE([GRID_COMMS_MPI3L],[1],[GRID_COMMS_MPI3L] )
-       comms_type='mpi3l'
-     ;;
      mpi3*)
         AC_DEFINE([GRID_COMMS_MPI3],[1],[GRID_COMMS_MPI3] )
         comms_type='mpi3'
      ;;
+     mpit)
+        AC_DEFINE([GRID_COMMS_MPIT],[1],[GRID_COMMS_MPIT] )
+        comms_type='mpit'
+     ;;
      mpi*)
         AC_DEFINE([GRID_COMMS_MPI],[1],[GRID_COMMS_MPI] )
         comms_type='mpi'
@@ -359,7 +359,7 @@ esac
 AM_CONDITIONAL(BUILD_COMMS_SHMEM, [ test "${comms_type}X" == "shmemX" ])
 AM_CONDITIONAL(BUILD_COMMS_MPI,   [ test "${comms_type}X" == "mpiX" ])
 AM_CONDITIONAL(BUILD_COMMS_MPI3,  [ test "${comms_type}X" == "mpi3X" ] )
-AM_CONDITIONAL(BUILD_COMMS_MPI3L, [ test "${comms_type}X" == "mpi3lX" ] )
+AM_CONDITIONAL(BUILD_COMMS_MPIT,  [ test "${comms_type}X" == "mpitX" ] )
 AM_CONDITIONAL(BUILD_COMMS_NONE,  [ test "${comms_type}X" == "noneX" ])
 
 ############### RNG selection
diff --git a/lib/Makefile.am b/lib/Makefile.am
index fac622ca..6dd7899e 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -10,8 +10,8 @@ if BUILD_COMMS_MPI3
   extra_sources+=communicator/Communicator_base.cc
 endif
 
-if BUILD_COMMS_MPI3L
-  extra_sources+=communicator/Communicator_mpi3_leader.cc
+if BUILD_COMMS_MPIT
+  extra_sources+=communicator/Communicator_mpit.cc
   extra_sources+=communicator/Communicator_base.cc
 endif
 
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 557fef48..a5edf8e9 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -89,25 +89,31 @@ void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
   GlobalSumVector((double *)c,2*N);
 }
 
-#if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPI3L)
+#if !defined( GRID_COMMS_MPI3) 
 
 int                      CartesianCommunicator::NodeCount(void)    { return ProcessorCount();};
 int                      CartesianCommunicator::RankCount(void)    { return ProcessorCount();};
-
+#endif
+#if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPIT)
 double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-						       void *xmit,
-						       int xmit_to_rank,
-						       void *recv,
-						       int recv_from_rank,
-						       int bytes)
+							 void *xmit,
+							 int xmit_to_rank,
+							 void *recv,
+							 int recv_from_rank,
+							 int bytes, int dir)
 {
+  // Discard the "dir"
   SendToRecvFromBegin(list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
   return 2.0*bytes;
 }
-void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall)
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
 {
   SendToRecvFromComplete(waitall);
 }
+#endif
+
+#if !defined( GRID_COMMS_MPI3) 
+
 void CartesianCommunicator::StencilBarrier(void){};
 
 commVector<uint8_t> CartesianCommunicator::ShmBufStorageVector;
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 12a8429f..4e471b43 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -38,7 +38,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifdef GRID_COMMS_MPI3
 #include <mpi.h>
 #endif
-#ifdef GRID_COMMS_MPI3L
+#ifdef GRID_COMMS_MPIT
 #include <mpi.h>
 #endif
 #ifdef GRID_COMMS_SHMEM
@@ -64,7 +64,7 @@ class CartesianCommunicator {
   std::vector<int> _processor_coor;  // linear processor coordinate
   unsigned long _ndimension;
 
-#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPI3L)
+#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPIT)
   static MPI_Comm communicator_world;
          MPI_Comm communicator;
   typedef MPI_Request CommsRequest_t;
@@ -72,6 +72,10 @@ class CartesianCommunicator {
   typedef int CommsRequest_t;
 #endif
 
+#if defined (GRID_COMMS_MPIT)
+  std::vector<MPI_Comm> communicator_halo;
+#endif
+
   ////////////////////////////////////////////////////////////////////
   // Helper functionality for SHM Windows common to all other impls
   ////////////////////////////////////////////////////////////////////
@@ -212,13 +216,13 @@ class CartesianCommunicator {
   void SendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
 
   double StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-				  void *xmit,
-				  int xmit_to_rank,
-				  void *recv,
-				  int recv_from_rank,
-				  int bytes);
+				    void *xmit,
+				    int xmit_to_rank,
+				    void *recv,
+				    int recv_from_rank,
+				    int bytes,int dir);
   
-  void StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
+  void StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int i);
   void StencilBarrier(void);
 
   ////////////////////////////////////////////////////////////
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 632eb991..8046fef6 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -600,11 +600,11 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
 }
 
 double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
-						       void *xmit,
-						       int dest,
-						       void *recv,
-						       int from,
-						       int bytes)
+							 void *xmit,
+							 int dest,
+							 void *recv,
+							 int from,
+							 int bytes,int dir)
 {
   MPI_Request xrq;
   MPI_Request rrq;
@@ -643,7 +643,7 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 
   return off_node_bytes;
 }
-void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall)
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
 {
   SendToRecvFromComplete(waitall);
 }
diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc
index 07522900..24a518ec 100644
--- a/lib/communicator/Communicator_mpit.cc
+++ b/lib/communicator/Communicator_mpit.cc
@@ -235,24 +235,30 @@ void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
   assert(ierr==0);
 }
 
-  double CartesianCommunicator::StencilSendToRecvFromBegin(int dir,
-							   std::vector<CommsRequest_t> &list,
-							   void *xmit,
-							   int xmit_to_rank,
-							   void *recv,
-							   int recv_from_rank,
-							   int bytes)
+double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
+							 void *xmit,
+							 int xmit_to_rank,
+							 void *recv,
+							 int recv_from_rank,
+							 int bytes,int dir)
 {
+
   int myrank = _processor;
   int ierr;
+  assert(dir < communicator_halo.size());
+
+  //  std::cout << " sending on communicator "<<dir<<" " <<communicator_halo[dir]<<std::endl;
   // Give the CPU to MPI immediately; can use threads to overlap optionally
-  ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,dest,myrank,
-		    recv,bytes,MPI_CHAR,from, from,
+  ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,xmit_to_rank,myrank,
+		    recv,bytes,MPI_CHAR,recv_from_rank, recv_from_rank,
 		    communicator_halo[dir],MPI_STATUS_IGNORE);
   assert(ierr==0);
   return 2.0*bytes;
 }
-void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall){ };
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
+{ 
+  // Do nothing
+};
 
 
 
diff --git a/lib/cshift/Cshift.h b/lib/cshift/Cshift.h
index cd162e35..7d0caeee 100644
--- a/lib/cshift/Cshift.h
+++ b/lib/cshift/Cshift.h
@@ -42,7 +42,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/cshift/Cshift_mpi.h>
 #endif 
 
-#ifdef GRID_COMMS_MPI3L
+#ifdef GRID_COMMS_MPIT
 #include <Grid/cshift/Cshift_mpi.h>
 #endif 
 
diff --git a/lib/log/Log.cc b/lib/log/Log.cc
index 69a9a0a8..65dc2812 100644
--- a/lib/log/Log.cc
+++ b/lib/log/Log.cc
@@ -95,7 +95,7 @@ void GridLogConfigure(std::vector<std::string> &logstreams) {
 ////////////////////////////////////////////////////////////
 void Grid_quiesce_nodes(void) {
   int me = 0;
-#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) || defined(GRID_COMMS_MPI3L)
+#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) || defined(GRID_COMMS_MPIT)
   MPI_Comm_rank(MPI_COMM_WORLD, &me);
 #endif
 #ifdef GRID_COMMS_SHMEM
diff --git a/lib/parallelIO/BinaryIO.h b/lib/parallelIO/BinaryIO.h
index 117bec01..480afa01 100644
--- a/lib/parallelIO/BinaryIO.h
+++ b/lib/parallelIO/BinaryIO.h
@@ -29,7 +29,7 @@
 #ifndef GRID_BINARY_IO_H
 #define GRID_BINARY_IO_H
 
-#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) 
+#if defined(GRID_COMMS_MPI) || defined(GRID_COMMS_MPI3) || defined(GRID_COMMS_MPIT) 
 #define USE_MPI_IO
 #else
 #undef  USE_MPI_IO
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index 27319fb0..6a6bc1f8 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -379,7 +379,6 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
 {
 #ifdef GRID_OMP
   //  assert((dag==DaggerNo) ||(dag==DaggerYes));
-  typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
 
   Compressor compressor(dag);
 
@@ -388,46 +387,46 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
 
   DhopFaceTime-=usecond();
   st.HaloExchangeOptGather(in,compressor);
+  st.CommsMergeSHM(compressor);// Could do this inside parallel region overlapped with comms
   DhopFaceTime+=usecond();
-  std::vector<std::vector<CommsRequest_t> > reqs;
 
   // Rely on async comms; start comms before merge of local data
+  DhopComputeTime-=usecond();
   DhopCommTime-=usecond();
-  st.CommunicateBegin(reqs);
-
-  DhopFaceTime-=usecond();
-  st.CommsMergeSHM(compressor);
-  DhopFaceTime+=usecond();
-
-  // Perhaps use omp task and region
 #pragma omp parallel 
   { 
-    int nthreads = omp_get_num_threads();
-    int me = omp_get_thread_num();
-    int myoff, mywork;
+    // Should time this somehow; hard as the threads fork nowait
+    st.CommunicateThreaded();
 
-    GridThread::GetWork(len,me-1,mywork,myoff,nthreads-1);
-    int sF = LLs * myoff;
-
-    if ( me == 0 ) {
-      st.CommunicateComplete(reqs);
-      DhopCommTime+=usecond();
-    } else { 
-      // Interior links in stencil
-      if ( me==1 ) DhopComputeTime-=usecond();
-      if (dag == DaggerYes) Kernels::DhopSiteDag(st,lo,U,st.CommBuf(),sF,myoff,LLs,mywork,in,out,1,0);
-      else      	    Kernels::DhopSite(st,lo,U,st.CommBuf(),sF,myoff,LLs,mywork,in,out,1,0);
-      if ( me==1 ) DhopComputeTime+=usecond();
+  if (dag == DaggerYes) {
+#pragma omp for
+    for (int ss = 0; ss < U._grid->oSites(); ss++) {
+      int sU = ss;
+      int sF = LLs * sU;
+      Kernels::DhopSiteDag(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,1,0);
+    }
+  } else {
+#pragma omp for
+    for (int ss = 0; ss < U._grid->oSites(); ss++) {
+      int sU = ss;
+      int sF = LLs * sU;
+      Kernels::DhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,1,0);
     }
   }
+#pragma omp single
+  DhopComputeTime+=usecond();
+
+#pragma omp taskwait 
+
+#pragma omp single
+  DhopCommTime+=usecond();
+  } // Closes parallel region and waits the comms (I hope)
+
 
   DhopFaceTime-=usecond();
   st.CommsMerge(compressor);
   DhopFaceTime+=usecond();
 
-  // Load imbalance alert. Should use dynamic schedule OMP for loop
-  // Perhaps create a list of only those sites with face work, and 
-  // load balance process the list.
   DhopComputeTime2-=usecond();
   if (dag == DaggerYes) {
     int sz=st.surface_list.size();
@@ -448,11 +447,9 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
 #else 
   assert(0);
 #endif
-
 }
 
 
-
 template<class Impl>
 void WilsonFermion5D<Impl>::DhopInternalSerialComms(StencilImpl & st, LebesgueOrder &lo,
 					 DoubledGaugeField & U,
diff --git a/lib/stencil/Stencil.h b/lib/stencil/Stencil.h
index 2894778a..17db64d8 100644
--- a/lib/stencil/Stencil.h
+++ b/lib/stencil/Stencil.h
@@ -248,24 +248,57 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
   //////////////////////////////////////////
   // Comms packet queue for asynch thread
   //////////////////////////////////////////
+  void CommunicateThreaded()
+  {
+    for(int i=0;i<Packets.size();i++){
+#pragma omp task 
+      {
+	double start;
+	double stop;
+	start = usecond();
+	uint64_t bytes;
+	std::vector<CommsRequest_t> reqs;
+	bytes=_grid->StencilSendToRecvFromBegin(reqs,
+					  Packets[i].send_buf,
+					  Packets[i].to_rank,
+					  Packets[i].recv_buf,
+					  Packets[i].from_rank,
+					  Packets[i].bytes,i);
+	_grid->StencilSendToRecvFromComplete(reqs,i);
+	// Last task logged; this is approximate but hard to catch
+	// the last to complete
+	stop = usecond();
+	stop = stop - start;
+
+	if ( i==0 ) commtime+=stop;
+
+#pragma omp critical
+	{
+	  comms_bytes+=bytes;
+	}
+
+      }
+    }
+    
+  }
   void CommunicateBegin(std::vector<std::vector<CommsRequest_t> > &reqs)
   {
     reqs.resize(Packets.size());
     commtime-=usecond();
     for(int i=0;i<Packets.size();i++){
       comms_bytes+=_grid->StencilSendToRecvFromBegin(reqs[i],
-					  Packets[i].send_buf,
-					  Packets[i].to_rank,
-					  Packets[i].recv_buf,
-					  Packets[i].from_rank,
-					  Packets[i].bytes);
+						     Packets[i].send_buf,
+						     Packets[i].to_rank,
+						     Packets[i].recv_buf,
+						     Packets[i].from_rank,
+						     Packets[i].bytes,i);
     }
   }
 
   void CommunicateComplete(std::vector<std::vector<CommsRequest_t> > &reqs)
   {
     for(int i=0;i<Packets.size();i++){
-      _grid->StencilSendToRecvFromComplete(reqs[i]);
+      _grid->StencilSendToRecvFromComplete(reqs[i],i);
     }
     commtime+=usecond();
   }
diff --git a/lib/util/Init.cc b/lib/util/Init.cc
index fe3b1734..fc701ac1 100644
--- a/lib/util/Init.cc
+++ b/lib/util/Init.cc
@@ -393,7 +393,7 @@ void Grid_init(int *argc,char ***argv)
 
 void Grid_finalize(void)
 {
-#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3)
+#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPIT)
   MPI_Finalize();
   Grid_unquiesce_nodes();
 #endif

From 6f5a5cd9b3269932a720804aebe8b7046d4b68fe Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 28 Jun 2017 23:27:02 +0100
Subject: [PATCH 015/115] Improved threaded comms benchmark

---
 TODO                          | 11 ++--
 benchmarks/Benchmark_comms.cc | 94 +++++++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+), 3 deletions(-)

diff --git a/TODO b/TODO
index 001c6c0c..3d29215e 100644
--- a/TODO
+++ b/TODO
@@ -2,10 +2,13 @@ TODO:
 ---------------
 
 Large item work list:
-1)- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O
+1)- I/O;    There appear to be issues with MPI IO and NERSC with large files.
+            Possible 2GB limit reappeared. GPFS driver in Intel MPI.
+
+2)- BG/Q port and check
+
+3)- Christoph's local basis expansion Lanczos; port to use Lattice_transfer features
 
-2)- Christoph's local basis expansion Lanczos
-3)- BG/Q port and check
 4)- Precision conversion and sort out localConvert      <-- partial
   - Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet
 5)- Physical propagator interface
@@ -14,6 +17,8 @@ Large item work list:
 8)- HDCR resume
 
 Recent DONE 
+
+-- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O.  <--- DONE
 -- Lanczos Remove DenseVector, DenseMatrix; Use Eigen instead. <-- DONE
 -- GaugeFix into central location                      <-- DONE
 -- Scidac and Ildg metadata handling                   <-- DONE
diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc
index 753b8a58..698f9d25 100644
--- a/benchmarks/Benchmark_comms.cc
+++ b/benchmarks/Benchmark_comms.cc
@@ -435,6 +435,100 @@ int main (int argc, char ** argv)
  
     }
   }    
+
+
+
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "= Benchmarking threaded STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
+  std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+  header();
+
+  for(int lat=4;lat<=maxlat;lat+=4){
+    for(int Ls=8;Ls<=8;Ls*=2){
+
+      std::vector<int> latt_size  ({lat*mpi_layout[0],
+      				    lat*mpi_layout[1],
+      				    lat*mpi_layout[2],
+      				    lat*mpi_layout[3]});
+
+      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+      RealD Nrank = Grid._Nprocessors;
+      RealD Nnode = Grid.NodeCount();
+      RealD ppn = Nrank/Nnode;
+
+      std::vector<HalfSpinColourVectorD *> xbuf(8);
+      std::vector<HalfSpinColourVectorD *> rbuf(8);
+      Grid.ShmBufferFreeAll();
+      for(int d=0;d<8;d++){
+	xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+      }
+
+      int ncomm;
+      int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+      double dbytes;
+      for(int i=0;i<Nloop;i++){
+	double start=usecond();
+
+	std::vector<CartesianCommunicator::CommsRequest_t> requests;
+	dbytes=0;
+	ncomm=0;
+
+	parallel_for(int dir=0;dir<8;dir++){
+
+	  double tbytes;
+	  int mu =dir % 4;
+
+	  if (mpi_layout[mu]>1 ) {
+	  
+	    ncomm++;
+	    int xmit_to_rank;
+	    int recv_from_rank;
+	    if ( dir == mu ) { 
+	      int comm_proc=1;
+	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    } else { 
+	      int comm_proc = mpi_layout[mu]-1;
+	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	    }
+	    tbytes= Grid.StencilSendToRecvFromBegin(requests,
+						    (void *)&xbuf[dir][0],
+						    xmit_to_rank,
+						    (void *)&rbuf[dir][0],
+						    recv_from_rank,
+						    bytes,dir);
+	    Grid.StencilSendToRecvFromComplete(requests,dir);
+	    requests.resize(0);
+
+#pragma omp atomic
+	    dbytes+=tbytes;
+	  }
+	}
+	Grid.Barrier();
+	double stop=usecond();
+	t_time[i] = stop-start; // microseconds
+      }
+
+      timestat.statistics(t_time);
+
+      dbytes=dbytes*ppn;
+      double xbytes    = dbytes*0.5;
+      double rbytes    = dbytes*0.5;
+      double bidibytes = dbytes;
+
+
+      std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+               <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+               <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+               <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+               << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+               << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
+ 
+    }
+  }    
+
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
   std::cout<<GridLogMessage << "= All done; Bye Bye"<<std::endl;
   std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;

From 57002924bc3232308cd1add6610b632a3b055fe9 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 2 Jul 2017 14:58:30 -0700
Subject: [PATCH 016/115] NERSC shakeout of this

---
 lib/communicator/Communicator_mpi.cc |  8 ++--
 tests/solver/Test_dwf_mrhs_cg.cc     | 65 ++++++++++++++--------------
 2 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index 7879f518..1a7a0c05 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -70,7 +70,7 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,
   MPI_Comm_size(parent.communicator,&Nparent);
 
   int childsize=1;
-  for(int d=0;d<_ndimension;d++) {
+  for(int d=0;d<processors.size();d++) {
     childsize *= processors[d];
   }
   int Nchild = Nparent/childsize;
@@ -83,12 +83,12 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,
   MPI_Comm comm_split;
   if ( Nchild > 1 ) { 
 
-    std::cout << GridLogMessage<<"Child communicator of "<< parent.communicator<<std::endl;
-    std::cout << GridLogMessage<<" parent grid    ";
+    std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec<<std::endl;
+    std::cout << GridLogMessage<<" parent grid["<< parent._ndimension<<"]    ";
     for(int d=0;d<parent._processors.size();d++)  std::cout << parent._processors[d] << " ";
     std::cout<<std::endl;
 
-    std::cout << GridLogMessage<<" child grid    ";
+    std::cout << GridLogMessage<<" child grid["<< _ndimension <<"]    ";
     for(int d=0;d<processors.size();d++)  std::cout << processors[d] << " ";
     std::cout<<std::endl;
 
diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc
index fa940390..e44c15d0 100644
--- a/tests/solver/Test_dwf_mrhs_cg.cc
+++ b/tests/solver/Test_dwf_mrhs_cg.cc
@@ -95,49 +95,50 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
   emptyUserRecord record;
   std::string file("./scratch.scidac");
-  {
-    ScidacWriter _ScidacWriter;
-    _ScidacWriter.open(file);
-    _ScidacWriter.writeScidacFieldRecord(Umu,record);
-    for(int n=0;n<nrhs;n++){
-      _ScidacWriter.writeScidacFieldRecord(src[n],record);
-      
-    }
-    _ScidacWriter.close();
-  }
-
-
-
-  //////////////////////////////////////////
-  // Read back into single rank fields
-  //////////////////////////////////////////
-
-  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
-  std::cout << GridLogMessage << " Reading back in the single process view "<<std::endl;
-  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
-
   int me = UGrid->ThisRank();
   LatticeGaugeField s_Umu(SGrid);
   FermionField s_src(SFGrid);
   FermionField s_res(SFGrid);
+
   {
+    ScidacWriter _ScidacWriter;
+    _ScidacWriter.open(file);
+    std::cout << GridLogMessage << " Writing out gauge field "<<std::endl;
+    _ScidacWriter.writeScidacFieldRecord(Umu,record);
+    _ScidacWriter.close();
+
+    std::cout << GridLogMessage << " Reading in gauge field "<<std::endl;
     ScidacReader  _ScidacReader;
     _ScidacReader.open(file);
-    std::cout << GridLogMessage << " Opened file "<<std::endl;
     _ScidacReader.readScidacFieldRecord(s_Umu,record);
-    std::cout << GridLogMessage << " Read gauge field "<<std::endl;
-    for(int n=0;n<nrhs;n++){
-      if ( n==me ) { 
-	std::cout << GridLogMessage << " Read record  "<<n<<std::endl;
-	_ScidacReader.readScidacFieldRecord(s_src,record);
-      } else { 
-	std::cout << GridLogMessage << " Skip record  "<<n<<std::endl;
-	_ScidacReader.skipScidacFieldRecord();
-      }      
-    }
     _ScidacReader.close();
   }
 
+
+  {
+    for(int n=0;n<nrhs;n++){
+
+      std::cout << GridLogMessage << " Writing out record "<<n<<std::endl;
+      ScidacWriter _ScidacWriter;
+      _ScidacWriter.open(file);
+      _ScidacWriter.writeScidacFieldRecord(src[n],record);
+      _ScidacWriter.close();
+      
+      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+      std::cout << GridLogMessage << " Reading back in the single process view "<<std::endl;
+      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+      
+      if ( n==me ) { 
+	ScidacReader  _ScidacReader;
+	_ScidacReader.open(file);
+	_ScidacReader.readScidacFieldRecord(s_src,record);
+	_ScidacReader.close();
+      }
+      FGrid->Barrier();
+    }
+  }
+
+
   ///////////////////////////////////////////////////////////////
   // Set up N-solvers as trivially parallel
   ///////////////////////////////////////////////////////////////

From dee68fc7287a2af2d2acba8f72f96760722d275a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 2 Jul 2017 23:33:48 +0100
Subject: [PATCH 017/115] IO working multiple nodes again. Strategy of all
 nodes writing metadata is unsafe. Only one rank should do this. must identify
 this rank. Means pass communicator to the Objects.

---
 lib/parallelIO/IldgIO.h          |  8 ++++++--
 lib/parallelIO/IldgIOtypes.h     |  1 +
 lib/parallelIO/MetaData.h        |  3 +++
 tests/solver/Test_dwf_mrhs_cg.cc | 21 ++++++++++++++++++---
 4 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h
index 542f80ef..7031f110 100644
--- a/lib/parallelIO/IldgIO.h
+++ b/lib/parallelIO/IldgIO.h
@@ -258,10 +258,13 @@ class GridLimeReader : public BinaryIO {
 
       if ( !strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) )  ) {
 
-	//	std::cout << GridLogMessage<< " readLimeObject matches ! " <<std::endl;
+	std::cout << GridLogMessage<< " readLimeObject matches ! " << record_name <<std::endl;
 
 	std::vector<char> xmlc(nbytes+1,'\0');
 	limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR);    
+
+	std::cout << GridLogMessage<< " readLimeObject matches XML " << &xmlc[0] <<std::endl;
+
 	XmlReader RD(&xmlc[0],"");
 	read(RD,object_name,object);
 	return;
@@ -276,8 +279,9 @@ class GridLimeWriter : public BinaryIO {
  public:
    ///////////////////////////////////////////////////
    // FIXME: format for RNG? Now just binary out instead
+   // FIXME: collective calls or not ?
+   //      : must know if I am the I/O boss
    ///////////////////////////////////////////////////
-
    FILE       *File;
    LimeWriter *LimeW;
    std::string filename;
diff --git a/lib/parallelIO/IldgIOtypes.h b/lib/parallelIO/IldgIOtypes.h
index 53664b49..5b397e14 100644
--- a/lib/parallelIO/IldgIOtypes.h
+++ b/lib/parallelIO/IldgIOtypes.h
@@ -66,6 +66,7 @@ namespace Grid {
 
 struct emptyUserRecord : Serializable { 
   GRID_SERIALIZABLE_CLASS_MEMBERS(emptyUserRecord,int,dummy);
+  emptyUserRecord() { dummy=0; };
 };
 
 ////////////////////////
diff --git a/lib/parallelIO/MetaData.h b/lib/parallelIO/MetaData.h
index 8fc0d777..ccc8b18f 100644
--- a/lib/parallelIO/MetaData.h
+++ b/lib/parallelIO/MetaData.h
@@ -85,6 +85,9 @@ namespace Grid {
 	nd=4;
 	dimension.resize(4);
 	boundary.resize(4);
+	scidac_checksuma=0;
+	scidac_checksumb=0;
+	checksum=0;
       }
     };
 
diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc
index e44c15d0..72d8d8f4 100644
--- a/tests/solver/Test_dwf_mrhs_cg.cc
+++ b/tests/solver/Test_dwf_mrhs_cg.cc
@@ -95,42 +95,57 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
   emptyUserRecord record;
   std::string file("./scratch.scidac");
+  std::string filef("./scratch.scidac.ferm");
   int me = UGrid->ThisRank();
   LatticeGaugeField s_Umu(SGrid);
   FermionField s_src(SFGrid);
   FermionField s_res(SFGrid);
 
   {
+    FGrid->Barrier();
     ScidacWriter _ScidacWriter;
     _ScidacWriter.open(file);
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     std::cout << GridLogMessage << " Writing out gauge field "<<std::endl;
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     _ScidacWriter.writeScidacFieldRecord(Umu,record);
     _ScidacWriter.close();
-
+    FGrid->Barrier();
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     std::cout << GridLogMessage << " Reading in gauge field "<<std::endl;
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
     ScidacReader  _ScidacReader;
     _ScidacReader.open(file);
     _ScidacReader.readScidacFieldRecord(s_Umu,record);
     _ScidacReader.close();
+    FGrid->Barrier();
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+    std::cout << GridLogMessage << " Read in gauge field "<<std::endl;
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
   }
 
 
   {
     for(int n=0;n<nrhs;n++){
 
+      FGrid->Barrier();
+      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       std::cout << GridLogMessage << " Writing out record "<<n<<std::endl;
+      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       ScidacWriter _ScidacWriter;
-      _ScidacWriter.open(file);
+      _ScidacWriter.open(filef);
       _ScidacWriter.writeScidacFieldRecord(src[n],record);
       _ScidacWriter.close();
       
+      FGrid->Barrier();
+
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       std::cout << GridLogMessage << " Reading back in the single process view "<<std::endl;
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       
       if ( n==me ) { 
 	ScidacReader  _ScidacReader;
-	_ScidacReader.open(file);
+	_ScidacReader.open(filef);
 	_ScidacReader.readScidacFieldRecord(s_src,record);
 	_ScidacReader.close();
       }

From 75dc7794b9aaf6cf6b27c4d288d8b11c8f57b901 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 2 Jul 2017 16:47:42 -0700
Subject: [PATCH 018/115] Working on Cori

---
 lib/parallelIO/IldgIO.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/parallelIO/IldgIO.h b/lib/parallelIO/IldgIO.h
index 7031f110..ba71153d 100644
--- a/lib/parallelIO/IldgIO.h
+++ b/lib/parallelIO/IldgIO.h
@@ -178,7 +178,7 @@ class GridLimeReader : public BinaryIO {
    /////////////////////////////////////////////
    // Open the file
    /////////////////////////////////////////////
-   void open(std::string &_filename) 
+   void open(const std::string &_filename) 
    {
      filename= _filename;
      File = fopen(filename.c_str(), "r");
@@ -258,12 +258,12 @@ class GridLimeReader : public BinaryIO {
 
       if ( !strncmp(limeReaderType(LimeR), record_name.c_str(),strlen(record_name.c_str()) )  ) {
 
-	std::cout << GridLogMessage<< " readLimeObject matches ! " << record_name <<std::endl;
+	//	std::cout << GridLogMessage<< " readLimeObject matches ! " << record_name <<std::endl;
 
 	std::vector<char> xmlc(nbytes+1,'\0');
 	limeReaderReadData((void *)&xmlc[0], &nbytes, LimeR);    
 
-	std::cout << GridLogMessage<< " readLimeObject matches XML " << &xmlc[0] <<std::endl;
+	//	std::cout << GridLogMessage<< " readLimeObject matches XML " << &xmlc[0] <<std::endl;
 
 	XmlReader RD(&xmlc[0],"");
 	read(RD,object_name,object);
@@ -286,7 +286,7 @@ class GridLimeWriter : public BinaryIO {
    LimeWriter *LimeW;
    std::string filename;
 
-   void open(std::string &_filename) { 
+   void open(const std::string &_filename) { 
      filename= _filename;
      File = fopen(filename.c_str(), "w");
      LimeW = limeCreateWriter(File); assert(LimeW != NULL );

From ac740f73ceb839f16b914dabb940119cc2003917 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 2 Jul 2017 16:47:58 -0700
Subject: [PATCH 019/115] Works on Cori

---
 tests/solver/Test_dwf_mrhs_cg.cc | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc
index 72d8d8f4..b4880148 100644
--- a/tests/solver/Test_dwf_mrhs_cg.cc
+++ b/tests/solver/Test_dwf_mrhs_cg.cc
@@ -128,29 +128,36 @@ int main (int argc, char ** argv)
   {
     for(int n=0;n<nrhs;n++){
 
-      FGrid->Barrier();
+      std::stringstream filefn;
+      filefn << filef << "."<< n;
+
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       std::cout << GridLogMessage << " Writing out record "<<n<<std::endl;
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       ScidacWriter _ScidacWriter;
-      _ScidacWriter.open(filef);
+      _ScidacWriter.open(filefn.str());
       _ScidacWriter.writeScidacFieldRecord(src[n],record);
       _ScidacWriter.close();
+    }
       
-      FGrid->Barrier();
+    FGrid->Barrier();
 
-      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
-      std::cout << GridLogMessage << " Reading back in the single process view "<<std::endl;
-      std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+    std::cout << GridLogMessage << " Reading back in the single process view "<<std::endl;
+    std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       
+    for(int n=0;n<nrhs;n++){
+
+      std::stringstream filefn;
+      filefn << filef << "."<< n;
       if ( n==me ) { 
 	ScidacReader  _ScidacReader;
-	_ScidacReader.open(filef);
+	_ScidacReader.open(filefn.str());
 	_ScidacReader.readScidacFieldRecord(s_src,record);
 	_ScidacReader.close();
       }
-      FGrid->Barrier();
     }
+    FGrid->Barrier();
   }
 
 
@@ -185,7 +192,7 @@ int main (int argc, char ** argv)
   // Report how long they all took
   /////////////////////////////////////////////////////////////
   for(int r=0;r<nrhs;r++){
-    std::cout << " Rank "<<r<<" "<< iterations[r]<<" CG iterations"<<std::endl;
+    std::cout << GridLogMessage<<" Rank "<<r<<" "<< iterations[r]<<" CG iterations"<<std::endl;
   }
   Grid_finalize();
 }

From 097c9637eede438c2264e94bb0dbc608de219169 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Tue, 11 Jul 2017 14:31:57 +0100
Subject: [PATCH 020/115] Fixed the JSON parsing error

---
 lib/json/json.hpp              | 9862 ++++++++++++++++++--------------
 lib/serialisation/BaseIO.h     |    2 +-
 lib/serialisation/JSON_IO.cc   |    2 +
 lib/serialisation/JSON_IO.h    |   29 +-
 tests/IO/Test_serialisation.cc |  131 +-
 5 files changed, 5815 insertions(+), 4211 deletions(-)

diff --git a/lib/json/json.hpp b/lib/json/json.hpp
index 6dfc1831..e53fbcf2 100644
--- a/lib/json/json.hpp
+++ b/lib/json/json.hpp
@@ -32,17 +32,17 @@ SOFTWARE.
 #include <algorithm> // all_of, copy, fill, find, for_each, none_of, remove, reverse, transform
 #include <array> // array
 #include <cassert> // assert
-#include <cctype> // isdigit
 #include <ciso646> // and, not, or
+#include <clocale> // lconv, localeconv
 #include <cmath> // isfinite, labs, ldexp, signbit
 #include <cstddef> // nullptr_t, ptrdiff_t, size_t
 #include <cstdint> // int64_t, uint64_t
 #include <cstdlib> // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull
-#include <cstring> // strlen
+#include <cstring> // memcpy, strlen
 #include <forward_list> // forward_list
 #include <functional> // function, hash, less
 #include <initializer_list> // initializer_list
-#include <iomanip> // setw
+#include <iomanip> // hex
 #include <iostream> // istream, ostream
 #include <iterator> // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
 #include <limits> // numeric_limits
@@ -51,7 +51,6 @@ SOFTWARE.
 #include <memory> // addressof, allocator, allocator_traits, unique_ptr
 #include <numeric> // accumulate
 #include <sstream> // stringstream
-#include <stdexcept> // domain_error, invalid_argument, out_of_range
 #include <string> // getline, stoi, string, to_string
 #include <type_traits> // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type
 #include <utility> // declval, forward, make_pair, move, pair, swap
@@ -90,7 +89,7 @@ SOFTWARE.
 #endif
 
 // allow to disable exceptions
-#if not defined(JSON_NOEXCEPTION) || defined(__EXCEPTIONS)
+#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION)
     #define JSON_THROW(exception) throw exception
     #define JSON_TRY try
     #define JSON_CATCH(exception) catch(exception)
@@ -100,6 +99,15 @@ SOFTWARE.
     #define JSON_CATCH(exception) if(false)
 #endif
 
+// manual branch prediction
+#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
+    #define JSON_LIKELY(x)      __builtin_expect(!!(x), 1)
+    #define JSON_UNLIKELY(x)    __builtin_expect(!!(x), 0)
+#else
+    #define JSON_LIKELY(x)      x
+    #define JSON_UNLIKELY(x)    x
+#endif
+
 /*!
 @brief namespace for Niels Lohmann
 @see https://github.com/nlohmann
@@ -118,6 +126,259 @@ This namespace collects some functions that could not be defined inside the
 */
 namespace detail
 {
+////////////////
+// exceptions //
+////////////////
+
+/*!
+@brief general exception of the @ref basic_json class
+
+Extension of std::exception objects with a member @a id for exception ids.
+
+@note To have nothrow-copy-constructible exceptions, we internally use
+      std::runtime_error which can cope with arbitrary-length error messages.
+      Intermediate strings are built with static functions and then passed to
+      the actual constructor.
+
+@since version 3.0.0
+*/
+class exception : public std::exception
+{
+  public:
+    /// returns the explanatory string
+    virtual const char* what() const noexcept override
+    {
+        return m.what();
+    }
+
+    /// the id of the exception
+    const int id;
+
+  protected:
+    exception(int id_, const char* what_arg)
+        : id(id_), m(what_arg)
+    {}
+
+    static std::string name(const std::string& ename, int id)
+    {
+        return "[json.exception." + ename + "." + std::to_string(id) + "] ";
+    }
+
+  private:
+    /// an exception object as storage for error messages
+    std::runtime_error m;
+};
+
+/*!
+@brief exception indicating a parse error
+
+This excpetion is thrown by the library when a parse error occurs. Parse
+errors can occur during the deserialization of JSON text as well as when
+using JSON Patch.
+
+Member @a byte holds the byte index of the last read character in the input
+file.
+
+@note For an input with n bytes, 1 is the index of the first character
+      and n+1 is the index of the terminating null byte or the end of
+      file. This also holds true when reading a byte vector (CBOR or
+      MessagePack).
+
+Exceptions have ids 1xx.
+
+name / id                      | example massage | description
+------------------------------ | --------------- | -------------------------
+json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position.
+json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point.
+json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid.
+json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects.
+json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors.
+json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number wihtout a leading `0`.
+json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character.
+json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences.
+json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number.
+json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read.
+json.exception.parse_error.111 | parse error: bad input stream | Parsing CBOR or MessagePack from an input stream where the [`badbit` or `failbit`](http://en.cppreference.com/w/cpp/io/ios_base/iostate) is set.
+json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xf8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
+json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read.
+
+@since version 3.0.0
+*/
+class parse_error : public exception
+{
+  public:
+    /*!
+    @brief create a parse error exception
+    @param[in] id         the id of the exception
+    @param[in] byte_      the byte index where the error occured (or 0 if
+                          the position cannot be determined)
+    @param[in] what_arg   the explanatory string
+    @return parse_error object
+    */
+    static parse_error create(int id, size_t byte_, const std::string& what_arg)
+    {
+        std::string w = exception::name("parse_error", id) + "parse error" +
+                        (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") +
+                        ": " + what_arg;
+        return parse_error(id, byte_, w.c_str());
+    }
+
+    /*!
+    @brief byte index of the parse error
+
+    The byte index of the last read character in the input file.
+
+    @note For an input with n bytes, 1 is the index of the first character
+          and n+1 is the index of the terminating null byte or the end of
+          file. This also holds true when reading a byte vector (CBOR or
+          MessagePack).
+    */
+    const size_t byte;
+
+  private:
+    parse_error(int id_, size_t byte_, const char* what_arg)
+        : exception(id_, what_arg), byte(byte_)
+    {}
+};
+
+/*!
+@brief exception indicating errors with iterators
+
+Exceptions have ids 2xx.
+
+name / id                           | example massage | description
+----------------------------------- | --------------- | -------------------------
+json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
+json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion.
+json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from.
+json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid.
+json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid.
+json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range.
+json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key.
+json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
+json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered.
+json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
+json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to.
+json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container.
+json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compated, because JSON objects are unordered.
+json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin().
+
+@since version 3.0.0
+*/
+class invalid_iterator : public exception
+{
+  public:
+    static invalid_iterator create(int id, const std::string& what_arg)
+    {
+        std::string w = exception::name("invalid_iterator", id) + what_arg;
+        return invalid_iterator(id, w.c_str());
+    }
+
+  private:
+    invalid_iterator(int id_, const char* what_arg)
+        : exception(id_, what_arg)
+    {}
+};
+
+/*!
+@brief exception indicating executing a member function with a wrong type
+
+Exceptions have ids 3xx.
+
+name / id                     | example message | description
+----------------------------- | --------------- | -------------------------
+json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead.
+json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types.
+json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&.
+json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types.
+json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types.
+json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types.
+json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types.
+json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types.
+json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types.
+json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types.
+json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types.
+json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined.
+json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers.
+json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive.
+
+@since version 3.0.0
+*/
+class type_error : public exception
+{
+  public:
+    static type_error create(int id, const std::string& what_arg)
+    {
+        std::string w = exception::name("type_error", id) + what_arg;
+        return type_error(id, w.c_str());
+    }
+
+  private:
+    type_error(int id_, const char* what_arg)
+        : exception(id_, what_arg)
+    {}
+};
+
+/*!
+@brief exception indicating access out of the defined range
+
+Exceptions have ids 4xx.
+
+name / id                       | example message | description
+------------------------------- | --------------- | -------------------------
+json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1.
+json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it.
+json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object.
+json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved.
+json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value.
+json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF.
+
+@since version 3.0.0
+*/
+class out_of_range : public exception
+{
+  public:
+    static out_of_range create(int id, const std::string& what_arg)
+    {
+        std::string w = exception::name("out_of_range", id) + what_arg;
+        return out_of_range(id, w.c_str());
+    }
+
+  private:
+    out_of_range(int id_, const char* what_arg)
+        : exception(id_, what_arg)
+    {}
+};
+
+/*!
+@brief exception indicating other errors
+
+Exceptions have ids 5xx.
+
+name / id                      | example message | description
+------------------------------ | --------------- | -------------------------
+json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed.
+json.exception.other_error.502 | invalid object size for conversion | Some conversions to user-defined types impose constraints on the object size (e.g. std::pair)
+
+@since version 3.0.0
+*/
+class other_error : public exception
+{
+  public:
+    static other_error create(int id, const std::string& what_arg)
+    {
+        std::string w = exception::name("other_error", id) + what_arg;
+        return other_error(id, w.c_str());
+    }
+
+  private:
+    other_error(int id_, const char* what_arg)
+        : exception(id_, what_arg)
+    {}
+};
+
+
+
 ///////////////////////////
 // JSON type enumeration //
 ///////////////////////////
@@ -204,11 +465,38 @@ using enable_if_t = typename std::enable_if<B, T>::type;
 template<typename T>
 using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type;
 
-// taken from http://stackoverflow.com/a/26936864/266378
-template<typename T>
-using is_unscoped_enum =
-    std::integral_constant<bool, std::is_convertible<T, int>::value and
-    std::is_enum<T>::value>;
+// implementation of C++14 index_sequence and affiliates
+// source: https://stackoverflow.com/a/32223343
+template <std::size_t... Ints>
+struct index_sequence
+{
+    using type = index_sequence;
+    using value_type = std::size_t;
+    static constexpr std::size_t size() noexcept
+    {
+        return sizeof...(Ints);
+    }
+};
+
+template <class Sequence1, class Sequence2>
+struct merge_and_renumber;
+
+template <std::size_t... I1, std::size_t... I2>
+struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
+        : index_sequence < I1..., (sizeof...(I1) + I2)... >
+          { };
+
+template <std::size_t N>
+struct make_index_sequence
+    : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
+      typename make_index_sequence < N - N / 2 >::type >
+{ };
+
+template<> struct make_index_sequence<0> : index_sequence<> { };
+template<> struct make_index_sequence<1> : index_sequence<0> { };
+
+template<typename... Ts>
+using index_sequence_for = make_index_sequence<sizeof...(Ts)>;
 
 /*
 Implementation of two C++17 constructs: conjunction, negation. This is needed
@@ -271,16 +559,8 @@ struct external_constructor<value_t::number_float>
     template<typename BasicJsonType>
     static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept
     {
-        // replace infinity and NAN by null
-        if (not std::isfinite(val))
-        {
-            j = BasicJsonType{};
-        }
-        else
-        {
-            j.m_type = value_t::number_float;
-            j.m_value = val;
-        }
+        j.m_type = value_t::number_float;
+        j.m_value = val;
         j.assert_invariant();
     }
 };
@@ -332,6 +612,19 @@ struct external_constructor<value_t::array>
         j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
         j.assert_invariant();
     }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, const std::vector<bool>& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = value_t::array;
+        j.m_value.array->reserve(arr.size());
+        for (bool x : arr)
+        {
+            j.m_value.array->push_back(x);
+        }
+        j.assert_invariant();
+    }
 };
 
 template<>
@@ -563,11 +856,18 @@ void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept
     external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val));
 }
 
-template<typename BasicJsonType, typename UnscopedEnumType,
-         enable_if_t<is_unscoped_enum<UnscopedEnumType>::value, int> = 0>
-void to_json(BasicJsonType& j, UnscopedEnumType e) noexcept
+template<typename BasicJsonType, typename EnumType,
+         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+void to_json(BasicJsonType& j, EnumType e) noexcept
 {
-    external_constructor<value_t::number_integer>::construct(j, e);
+    using underlying_type = typename std::underlying_type<EnumType>::type;
+    external_constructor<value_t::number_integer>::construct(j, static_cast<underlying_type>(e));
+}
+
+template<typename BasicJsonType>
+void to_json(BasicJsonType& j, const std::vector<bool>& e)
+{
+    external_constructor<value_t::array>::construct(j, e);
 }
 
 template <
@@ -590,6 +890,32 @@ void to_json(BasicJsonType& j, const  CompatibleObjectType& arr)
     external_constructor<value_t::object>::construct(j, arr);
 }
 
+template <typename BasicJsonType, typename T, std::size_t N,
+          enable_if_t<not std::is_constructible<
+                          typename BasicJsonType::string_t, T (&)[N]>::value,
+                      int> = 0>
+void to_json(BasicJsonType& j, T (&arr)[N])
+{
+    external_constructor<value_t::array>::construct(j, arr);
+}
+
+template <typename BasicJsonType, typename... Args>
+void to_json(BasicJsonType& j, const std::pair<Args...>& p)
+{
+    j = {p.first, p.second};
+}
+
+template <typename BasicJsonType, typename Tuple, std::size_t... Idx>
+void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...>)
+{
+    j = {std::get<Idx>(t)...};
+}
+
+template <typename BasicJsonType, typename... Args>
+void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
+{
+    to_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+}
 
 ///////////////
 // from_json //
@@ -625,8 +951,7 @@ void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
         }
         default:
         {
-            JSON_THROW(
-                std::domain_error("type must be number, but is " + j.type_name()));
+            JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name()));
         }
     }
 }
@@ -636,7 +961,7 @@ void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
 {
     if (not j.is_boolean())
     {
-        JSON_THROW(std::domain_error("type must be boolean, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be boolean, but is " + j.type_name()));
     }
     b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();
 }
@@ -646,7 +971,7 @@ void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
 {
     if (not j.is_string())
     {
-        JSON_THROW(std::domain_error("type must be string, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be string, but is " + j.type_name()));
     }
     s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
 }
@@ -669,13 +994,13 @@ void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t&
     get_arithmetic_value(j, val);
 }
 
-template<typename BasicJsonType, typename UnscopedEnumType,
-         enable_if_t<is_unscoped_enum<UnscopedEnumType>::value, int> = 0>
-void from_json(const BasicJsonType& j, UnscopedEnumType& e)
+template<typename BasicJsonType, typename EnumType,
+         enable_if_t<std::is_enum<EnumType>::value, int> = 0>
+void from_json(const BasicJsonType& j, EnumType& e)
 {
-    typename std::underlying_type<UnscopedEnumType>::type val;
+    typename std::underlying_type<EnumType>::type val;
     get_arithmetic_value(j, val);
-    e = static_cast<UnscopedEnumType>(val);
+    e = static_cast<EnumType>(val);
 }
 
 template<typename BasicJsonType>
@@ -683,28 +1008,21 @@ void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr)
 {
     if (not j.is_array())
     {
-        JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name()));
     }
     arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
 }
 
 // forward_list doesn't have an insert method
-template<typename BasicJsonType, typename T, typename Allocator>
+template<typename BasicJsonType, typename T, typename Allocator,
+         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
 void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)
 {
-    // do not perform the check when user wants to retrieve jsons
-    // (except when it's null.. ?)
-    if (j.is_null())
+    if (not j.is_array())
     {
-        JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
-    }
-    if (not std::is_same<T, BasicJsonType>::value)
-    {
-        if (not j.is_array())
-        {
-            JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
-        }
+        JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name()));
     }
+
     for (auto it = j.rbegin(), end = j.rend(); it != end; ++it)
     {
         l.push_front(it->template get<T>());
@@ -736,8 +1054,8 @@ auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, prio
     using std::end;
 
     arr.reserve(j.size());
-    std::transform(
-        j.begin(), j.end(), std::inserter(arr, end(arr)), [](const BasicJsonType & i)
+    std::transform(j.begin(), j.end(),
+                   std::inserter(arr, end(arr)), [](const BasicJsonType & i)
     {
         // get<BasicJsonType>() returns *this, this won't call a from_json
         // method when value_type is BasicJsonType
@@ -745,25 +1063,27 @@ auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, prio
     });
 }
 
+template <typename BasicJsonType, typename T, std::size_t N>
+void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2>)
+{
+    for (std::size_t i = 0; i < N; ++i)
+    {
+        arr[i] = j.at(i).template get<T>();
+    }
+}
+
 template<typename BasicJsonType, typename CompatibleArrayType,
          enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value and
+                     std::is_convertible<BasicJsonType, typename CompatibleArrayType::value_type>::value and
                      not std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, int> = 0>
 void from_json(const BasicJsonType& j, CompatibleArrayType& arr)
 {
-    if (j.is_null())
+    if (not j.is_array())
     {
-        JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name()));
     }
 
-    // when T == BasicJsonType, do not check if value_t is correct
-    if (not std::is_same<typename CompatibleArrayType::value_type, BasicJsonType>::value)
-    {
-        if (not j.is_array())
-        {
-            JSON_THROW(std::domain_error("type must be array, but is " + j.type_name()));
-        }
-    }
-    from_json_array_impl(j, arr, priority_tag<1> {});
+    from_json_array_impl(j, arr, priority_tag<2> {});
 }
 
 template<typename BasicJsonType, typename CompatibleObjectType,
@@ -772,16 +1092,23 @@ void from_json(const BasicJsonType& j, CompatibleObjectType& obj)
 {
     if (not j.is_object())
     {
-        JSON_THROW(std::domain_error("type must be object, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be object, but is " + j.type_name()));
     }
 
     auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
     using std::begin;
     using std::end;
-    // we could avoid the assignment, but this might require a for loop, which
-    // might be less efficient than the container constructor for some
-    // containers (would it?)
-    obj = CompatibleObjectType(begin(*inner_object), end(*inner_object));
+    using value_type = typename CompatibleObjectType::value_type;
+    std::transform(
+        inner_object->begin(), inner_object->end(),
+        std::inserter(obj, obj.begin()),
+        [](typename BasicJsonType::object_t::value_type const & p)
+    {
+        return value_type(
+                   p.first,
+                   p.second
+                   .template get<typename CompatibleObjectType::mapped_type>());
+    });
 }
 
 // overload for arithmetic types, not chosen for basic_json template arguments
@@ -822,11 +1149,29 @@ void from_json(const BasicJsonType& j, ArithmeticType& val)
         }
         default:
         {
-            JSON_THROW(std::domain_error("type must be number, but is " + j.type_name()));
+            JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name()));
         }
     }
 }
 
+template <typename BasicJsonType, typename... Args>
+void from_json(const BasicJsonType& j, std::pair<Args...>& p)
+{
+    p = {j.at(0), j.at(1)};
+}
+
+template <typename BasicJsonType, typename Tuple, std::size_t... Idx>
+void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...>)
+{
+    t = std::make_tuple(j.at(Idx)...);
+}
+
+template <typename BasicJsonType, typename... Args>
+void from_json(const BasicJsonType& j, std::tuple<Args...>& t)
+{
+    from_json_tuple_impl(j, t, index_sequence_for<Args...> {});
+}
+
 struct to_json_fn
 {
   private:
@@ -1054,6 +1399,31 @@ class basic_json
     template<typename T, typename SFINAE>
     using json_serializer = JSONSerializer<T, SFINAE>;
 
+
+    ////////////////
+    // exceptions //
+    ////////////////
+
+    /// @name exceptions
+    /// Classes to implement user-defined exceptions.
+    /// @{
+
+    /// @copydoc detail::exception
+    using exception = detail::exception;
+    /// @copydoc detail::parse_error
+    using parse_error = detail::parse_error;
+    /// @copydoc detail::invalid_iterator
+    using invalid_iterator = detail::invalid_iterator;
+    /// @copydoc detail::type_error
+    using type_error = detail::type_error;
+    /// @copydoc detail::out_of_range
+    using out_of_range = detail::out_of_range;
+    /// @copydoc detail::other_error
+    using other_error = detail::other_error;
+
+    /// @}
+
+
     /////////////////////
     // container types //
     /////////////////////
@@ -1136,10 +1506,7 @@ class basic_json
         result["url"] = "https://github.com/nlohmann/json";
         result["version"] =
         {
-            {"string", "2.1.1"},
-            {"major", 2},
-            {"minor", 1},
-            {"patch", 1}
+            {"string", "2.1.1"}, {"major", 2}, {"minor", 1}, {"patch", 1}
         };
 
 #ifdef _WIN32
@@ -1745,9 +2112,9 @@ class basic_json
 
                 default:
                 {
-                    if (t == value_t::null)
+                    if (JSON_UNLIKELY(t == value_t::null))
                     {
-                        JSON_THROW(std::domain_error("961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE
+                        JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 2.1.1")); // LCOV_EXCL_LINE
                     }
                     break;
                 }
@@ -1905,9 +2272,6 @@ class basic_json
 
     @complexity Constant.
 
-    @throw std::bad_alloc if allocation for object, array, or string value
-    fails
-
     @liveexample{The following code shows the constructor for different @ref
     value_t values,basic_json__value_t}
 
@@ -2062,10 +2426,12 @@ class basic_json
     value_t::array and @ref value_t::object are valid); when @a type_deduction
     is set to `true`, this parameter has no effect
 
-    @throw std::domain_error if @a type_deduction is `false`, @a manual_type
-    is `value_t::object`, but @a init contains an element which is not a pair
-    whose first element is a string; example: `"cannot create object from
-    initializer list"`
+    @throw type_error.301 if @a type_deduction is `false`, @a manual_type is
+    `value_t::object`, but @a init contains an element which is not a pair
+    whose first element is a string. In this case, the constructor could not
+    create an object. If @a type_deduction would have be `true`, an array
+    would have been created. See @ref object(std::initializer_list<basic_json>)
+    for an example.
 
     @complexity Linear in the size of the initializer list @a init.
 
@@ -2103,7 +2469,7 @@ class basic_json
             // if object is wanted but impossible, throw an exception
             if (manual_type == value_t::object and not is_an_object)
             {
-                JSON_THROW(std::domain_error("cannot create object from initializer list"));
+                JSON_THROW(type_error::create(301, "cannot create object from initializer list"));
             }
         }
 
@@ -2179,16 +2545,17 @@ class basic_json
     related function @ref array(std::initializer_list<basic_json>), there are
     no cases which can only be expressed by this function. That is, any
     initializer list @a init can also be passed to the initializer list
-    constructor @ref basic_json(std::initializer_list<basic_json>, bool,
-    value_t).
+    constructor @ref basic_json(std::initializer_list<basic_json>, bool, value_t).
 
     @param[in] init  initializer list to create an object from (optional)
 
     @return JSON object value
 
-    @throw std::domain_error if @a init is not a pair whose first elements are
-    strings; thrown by
-    @ref basic_json(std::initializer_list<basic_json>, bool, value_t)
+    @throw type_error.301 if @a init is not a list of pairs whose first
+    elements are strings. In this case, no object can be created. When such a
+    value is passed to @ref basic_json(std::initializer_list<basic_json>, bool, value_t),
+    an array would have been created from the passed initializer list @a init.
+    See example below.
 
     @complexity Linear in the size of @a init.
 
@@ -2240,10 +2607,10 @@ class basic_json
     The semantics depends on the different types a JSON value can have:
     - In case of primitive types (number, boolean, or string), @a first must
       be `begin()` and @a last must be `end()`. In this case, the value is
-      copied. Otherwise, std::out_of_range is thrown.
+      copied. Otherwise, invalid_iterator.204 is thrown.
     - In case of structured types (array, object), the constructor behaves as
       similar versions for `std::vector`.
-    - In case of a null type, std::domain_error is thrown.
+    - In case of a null type, invalid_iterator.206 is thrown.
 
     @tparam InputIT an input iterator type (@ref iterator or @ref
     const_iterator)
@@ -2254,14 +2621,19 @@ class basic_json
     @pre Iterators @a first and @a last must be initialized. **This
          precondition is enforced with an assertion.**
 
-    @throw std::domain_error if iterators are not compatible; that is, do not
-    belong to the same JSON value; example: `"iterators are not compatible"`
-    @throw std::out_of_range if iterators are for a primitive type (number,
-    boolean, or string) where an out of range error can be detected easily;
-    example: `"iterators out of range"`
-    @throw std::bad_alloc if allocation for object, array, or string fails
-    @throw std::domain_error if called with a null value; example: `"cannot
-    use construct with iterators from null"`
+    @pre Range `[first, last)` is valid. Usually, this precondition cannot be
+         checked efficiently. Only certain edge cases are detected; see the
+         description of the exceptions below.
+
+    @throw invalid_iterator.201 if iterators @a first and @a last are not
+    compatible (i.e., do not belong to the same JSON value). In this case,
+    the range `[first, last)` is undefined.
+    @throw invalid_iterator.204 if iterators @a first and @a last belong to a
+    primitive type (number, boolean, or string), but @a first does not point
+    to the first element any more. In this case, the range `[first, last)` is
+    undefined. See example code below.
+    @throw invalid_iterator.206 if iterators @a first and @a last belong to a
+    null value. In this case, the range `[first, last)` is undefined.
 
     @complexity Linear in distance between @a first and @a last.
 
@@ -2281,7 +2653,7 @@ class basic_json
         // make sure iterator fits the current value
         if (first.m_object != last.m_object)
         {
-            JSON_THROW(std::domain_error("iterators are not compatible"));
+            JSON_THROW(invalid_iterator::create(201, "iterators are not compatible"));
         }
 
         // copy type from first iterator
@@ -2298,7 +2670,7 @@ class basic_json
             {
                 if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end())
                 {
-                    JSON_THROW(std::out_of_range("iterators out of range"));
+                    JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
                 }
                 break;
             }
@@ -2357,47 +2729,14 @@ class basic_json
 
             default:
             {
-                JSON_THROW(std::domain_error("cannot use construct with iterators from " + first.m_object->type_name()));
+                JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " +
+                                                    first.m_object->type_name()));
             }
         }
 
         assert_invariant();
     }
 
-    /*!
-    @brief construct a JSON value given an input stream
-
-    @param[in,out] i  stream to read a serialized JSON value from
-    @param[in] cb a parser callback function of type @ref parser_callback_t
-    which is used to control the deserialization by filtering unwanted values
-    (optional)
-
-    @complexity Linear in the length of the input. The parser is a predictive
-    LL(1) parser. The complexity can be higher if the parser callback function
-    @a cb has a super-linear complexity.
-
-    @note A UTF-8 byte order mark is silently ignored.
-
-    @deprecated This constructor is deprecated and will be removed in version
-      3.0.0 to unify the interface of the library. Deserialization will be
-      done by stream operators or by calling one of the `parse` functions,
-      e.g. @ref parse(std::istream&, const parser_callback_t). That is, calls
-      like `json j(i);` for an input stream @a i need to be replaced by
-      `json j = json::parse(i);`. See the example below.
-
-    @liveexample{The example below demonstrates constructing a JSON value from
-    a `std::stringstream` with and without callback
-    function.,basic_json__istream}
-
-    @since version 2.0.0, deprecated in version 2.0.3, to be removed in
-           version 3.0.0
-    */
-    JSON_DEPRECATED
-    explicit basic_json(std::istream& i, const parser_callback_t cb = nullptr)
-    {
-        *this = parser(i, cb).parse();
-        assert_invariant();
-    }
 
     ///////////////////////////////////////
     // other constructors and destructor //
@@ -2418,8 +2757,6 @@ class basic_json
     - The complexity is linear.
     - As postcondition, it holds: `other == basic_json(other)`.
 
-    @throw std::bad_alloc if allocation for object, array, or string fails.
-
     @liveexample{The following code shows an example for the copy
     constructor.,basic_json__basic_json}
 
@@ -2632,6 +2969,8 @@ class basic_json
     members will be pretty-printed with that indent level. An indent level of
     `0` will only insert newlines. `-1` (the default) selects the most compact
     representation.
+    @param[in] indent_char The character to use for indentation of @a indent is
+    greate than `0`. The default is ` ` (space).
 
     @return string containing the serialization of the JSON value
 
@@ -2642,22 +2981,23 @@ class basic_json
 
     @see https://docs.python.org/2/library/json.html#json.dump
 
-    @since version 1.0.0
+    @since version 1.0.0; indentaction character added in version 3.0.0
     */
-    string_t dump(const int indent = -1) const
+    string_t dump(const int indent = -1, const char indent_char = ' ') const
     {
-        std::stringstream ss;
+        string_t result;
+        serializer s(output_adapter<char>::create(result), indent_char);
 
         if (indent >= 0)
         {
-            dump(ss, true, static_cast<unsigned int>(indent));
+            s.dump(*this, true, static_cast<unsigned int>(indent));
         }
         else
         {
-            dump(ss, false, 0);
+            s.dump(*this, false, 0);
         }
 
-        return ss.str();
+        return result;
     }
 
     /*!
@@ -3030,7 +3370,7 @@ class basic_json
             return m_value.boolean;
         }
 
-        JSON_THROW(std::domain_error("type must be boolean, but is " + type_name()));
+        JSON_THROW(type_error::create(302, "type must be boolean, but is " + type_name()));
     }
 
     /// get a pointer to the value (object)
@@ -3125,7 +3465,7 @@ class basic_json
 
     @tparam ThisType will be deduced as `basic_json` or `const basic_json`
 
-    @throw std::domain_error if ReferenceType does not match underlying value
+    @throw type_error.303 if ReferenceType does not match underlying value
     type of the current JSON
     */
     template<typename ReferenceType, typename ThisType>
@@ -3142,8 +3482,7 @@ class basic_json
             return *ptr;
         }
 
-        JSON_THROW(std::domain_error("incompatible ReferenceType for get_ref, actual type is " +
-                                     obj.type_name()));
+        JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + obj.type_name()));
     }
 
   public:
@@ -3194,9 +3533,9 @@ class basic_json
     This overloads is chosen if:
     - @a ValueType is not @ref basic_json,
     - @ref json_serializer<ValueType> has a `from_json()` method of the form
-      `void from_json(const @ref basic_json&, ValueType&)`, and
+      `void from_json(const basic_json&, ValueType&)`, and
     - @ref json_serializer<ValueType> does not have a `from_json()` method of
-      the form `ValueType from_json(const @ref basic_json&)`
+      the form `ValueType from_json(const basic_json&)`
 
     @tparam ValueTypeCV the provided value type
     @tparam ValueType the returned value type
@@ -3255,7 +3594,7 @@ class basic_json
     This overloads is chosen if:
     - @a ValueType is not @ref basic_json and
     - @ref json_serializer<ValueType> has a `from_json()` method of the form
-      `ValueType from_json(const @ref basic_json&)`
+      `ValueType from_json(const basic_json&)`
 
     @note If @ref json_serializer<ValueType> has both overloads of
     `from_json()`, this one is chosen.
@@ -3422,10 +3761,10 @@ class basic_json
 
     @return reference to the internally stored JSON value if the requested
     reference type @a ReferenceType fits to the JSON value; throws
-    std::domain_error otherwise
+    type_error.303 otherwise
 
-    @throw std::domain_error in case passed type @a ReferenceType is
-    incompatible with the stored JSON value
+    @throw type_error.303 in case passed type @a ReferenceType is incompatible
+    with the stored JSON value; see example below
 
     @complexity Constant.
 
@@ -3468,8 +3807,9 @@ class basic_json
 
     @return copy of the JSON value, converted to type @a ValueType
 
-    @throw std::domain_error in case passed type @a ValueType is incompatible
-    to JSON, thrown by @ref get() const
+    @throw type_error.302 in case passed type @a ValueType is incompatible
+    to the JSON value type (e.g., the JSON value is of type boolean, but a
+    string is requested); see example below
 
     @complexity Linear in the size of the JSON value.
 
@@ -3487,6 +3827,9 @@ class basic_json
                    not std::is_same<ValueType, typename string_t::value_type>::value
 #ifndef _MSC_VER  // fix for issue #167 operator<< ambiguity under VS2015
                    and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value
+#endif
+#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_MSC_VER) && _MSC_VER >1900 && defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464
+                   and not std::is_same<ValueType, typename std::string_view>::value
 #endif
                    , int >::type = 0 >
     operator ValueType() const
@@ -3516,17 +3859,21 @@ class basic_json
 
     @return reference to the element at index @a idx
 
-    @throw std::domain_error if the JSON value is not an array; example:
-    `"cannot use at() with string"`
-    @throw std::out_of_range if the index @a idx is out of range of the array;
-    that is, `idx >= size()`; example: `"array index 7 is out of range"`
+    @throw type_error.304 if the JSON value is not an array; in this case,
+    calling `at` with an index makes no sense. See example below.
+    @throw out_of_range.401 if the index @a idx is out of range of the array;
+    that is, `idx >= size()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
 
     @complexity Constant.
 
-    @liveexample{The example below shows how array elements can be read and
-    written using `at()`.,at__size_type}
-
     @since version 1.0.0
+
+    @liveexample{The example below shows how array elements can be read and
+    written using `at()`. It also demonstrates the different exceptions that
+    can be thrown.,at__size_type}
     */
     reference at(size_type idx)
     {
@@ -3540,12 +3887,12 @@ class basic_json
             JSON_CATCH (std::out_of_range&)
             {
                 // create better exception explanation
-                JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range"));
+                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
             }
         }
         else
         {
-            JSON_THROW(std::domain_error("cannot use at() with " + type_name()));
+            JSON_THROW(type_error::create(304, "cannot use at() with " + type_name()));
         }
     }
 
@@ -3559,17 +3906,21 @@ class basic_json
 
     @return const reference to the element at index @a idx
 
-    @throw std::domain_error if the JSON value is not an array; example:
-    `"cannot use at() with string"`
-    @throw std::out_of_range if the index @a idx is out of range of the array;
-    that is, `idx >= size()`; example: `"array index 7 is out of range"`
+    @throw type_error.304 if the JSON value is not an array; in this case,
+    calling `at` with an index makes no sense. See example below.
+    @throw out_of_range.401 if the index @a idx is out of range of the array;
+    that is, `idx >= size()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
 
     @complexity Constant.
 
-    @liveexample{The example below shows how array elements can be read using
-    `at()`.,at__size_type_const}
-
     @since version 1.0.0
+
+    @liveexample{The example below shows how array elements can be read using
+    `at()`. It also demonstrates the different exceptions that can be thrown.,
+    at__size_type_const}
     */
     const_reference at(size_type idx) const
     {
@@ -3583,12 +3934,12 @@ class basic_json
             JSON_CATCH (std::out_of_range&)
             {
                 // create better exception explanation
-                JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range"));
+                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
             }
         }
         else
         {
-            JSON_THROW(std::domain_error("cannot use at() with " + type_name()));
+            JSON_THROW(type_error::create(304, "cannot use at() with " + type_name()));
         }
     }
 
@@ -3602,21 +3953,25 @@ class basic_json
 
     @return reference to the element at key @a key
 
-    @throw std::domain_error if the JSON value is not an object; example:
-    `"cannot use at() with boolean"`
-    @throw std::out_of_range if the key @a key is is not stored in the object;
-    that is, `find(key) == end()`; example: `"key "the fast" not found"`
+    @throw type_error.304 if the JSON value is not an object; in this case,
+    calling `at` with a key makes no sense. See example below.
+    @throw out_of_range.403 if the key @a key is is not stored in the object;
+    that is, `find(key) == end()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
 
     @complexity Logarithmic in the size of the container.
 
-    @liveexample{The example below shows how object elements can be read and
-    written using `at()`.,at__object_t_key_type}
-
     @sa @ref operator[](const typename object_t::key_type&) for unchecked
     access by reference
     @sa @ref value() for access by value with a default value
 
     @since version 1.0.0
+
+    @liveexample{The example below shows how object elements can be read and
+    written using `at()`. It also demonstrates the different exceptions that
+    can be thrown.,at__object_t_key_type}
     */
     reference at(const typename object_t::key_type& key)
     {
@@ -3630,12 +3985,12 @@ class basic_json
             JSON_CATCH (std::out_of_range&)
             {
                 // create better exception explanation
-                JSON_THROW(std::out_of_range("key '" + key + "' not found"));
+                JSON_THROW(out_of_range::create(403, "key '" + key + "' not found"));
             }
         }
         else
         {
-            JSON_THROW(std::domain_error("cannot use at() with " + type_name()));
+            JSON_THROW(type_error::create(304, "cannot use at() with " + type_name()));
         }
     }
 
@@ -3649,21 +4004,25 @@ class basic_json
 
     @return const reference to the element at key @a key
 
-    @throw std::domain_error if the JSON value is not an object; example:
-    `"cannot use at() with boolean"`
-    @throw std::out_of_range if the key @a key is is not stored in the object;
-    that is, `find(key) == end()`; example: `"key "the fast" not found"`
+    @throw type_error.304 if the JSON value is not an object; in this case,
+    calling `at` with a key makes no sense. See example below.
+    @throw out_of_range.403 if the key @a key is is not stored in the object;
+    that is, `find(key) == end()`. See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
 
     @complexity Logarithmic in the size of the container.
 
-    @liveexample{The example below shows how object elements can be read using
-    `at()`.,at__object_t_key_type_const}
-
     @sa @ref operator[](const typename object_t::key_type&) for unchecked
     access by reference
     @sa @ref value() for access by value with a default value
 
     @since version 1.0.0
+
+    @liveexample{The example below shows how object elements can be read using
+    `at()`. It also demonstrates the different exceptions that can be thrown.,
+    at__object_t_key_type_const}
     */
     const_reference at(const typename object_t::key_type& key) const
     {
@@ -3677,12 +4036,12 @@ class basic_json
             JSON_CATCH (std::out_of_range&)
             {
                 // create better exception explanation
-                JSON_THROW(std::out_of_range("key '" + key + "' not found"));
+                JSON_THROW(out_of_range::create(403, "key '" + key + "' not found"));
             }
         }
         else
         {
-            JSON_THROW(std::domain_error("cannot use at() with " + type_name()));
+            JSON_THROW(type_error::create(304, "cannot use at() with " + type_name()));
         }
     }
 
@@ -3699,8 +4058,8 @@ class basic_json
 
     @return reference to the element at index @a idx
 
-    @throw std::domain_error if JSON is not an array or null; example:
-    `"cannot use operator[] with string"`
+    @throw type_error.305 if the JSON value is not an array or null; in that
+    cases, using the [] operator with an index makes no sense.
 
     @complexity Constant if @a idx is in the range of the array. Otherwise
     linear in `idx - size()`.
@@ -3735,7 +4094,7 @@ class basic_json
             return m_value.array->operator[](idx);
         }
 
-        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3747,8 +4106,8 @@ class basic_json
 
     @return const reference to the element at index @a idx
 
-    @throw std::domain_error if JSON is not an array; example: `"cannot use
-    operator[] with null"`
+    @throw type_error.305 if the JSON value is not an array; in that cases,
+    using the [] operator with an index makes no sense.
 
     @complexity Constant.
 
@@ -3765,7 +4124,7 @@ class basic_json
             return m_value.array->operator[](idx);
         }
 
-        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3781,8 +4140,8 @@ class basic_json
 
     @return reference to the element at key @a key
 
-    @throw std::domain_error if JSON is not an object or null; example:
-    `"cannot use operator[] with string"`
+    @throw type_error.305 if the JSON value is not an object or null; in that
+    cases, using the [] operator with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
 
@@ -3811,7 +4170,7 @@ class basic_json
             return m_value.object->operator[](key);
         }
 
-        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3830,8 +4189,8 @@ class basic_json
     @pre The element with key @a key must exist. **This precondition is
          enforced with an assertion.**
 
-    @throw std::domain_error if JSON is not an object; example: `"cannot use
-    operator[] with null"`
+    @throw type_error.305 if the JSON value is not an object; in that cases,
+    using the [] operator with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
 
@@ -3853,7 +4212,7 @@ class basic_json
             return m_value.object->find(key)->second;
         }
 
-        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3869,8 +4228,8 @@ class basic_json
 
     @return reference to the element at key @a key
 
-    @throw std::domain_error if JSON is not an object or null; example:
-    `"cannot use operator[] with string"`
+    @throw type_error.305 if the JSON value is not an object or null; in that
+    cases, using the [] operator with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
 
@@ -3904,8 +4263,8 @@ class basic_json
 
     @return const reference to the element at key @a key
 
-    @throw std::domain_error if JSON is not an object; example: `"cannot use
-    operator[] with null"`
+    @throw type_error.305 if the JSON value is not an object; in that cases,
+    using the [] operator with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
 
@@ -3937,8 +4296,8 @@ class basic_json
 
     @return reference to the element at key @a key
 
-    @throw std::domain_error if JSON is not an object or null; example:
-    `"cannot use operator[] with string"`
+    @throw type_error.305 if the JSON value is not an object or null; in that
+    cases, using the [] operator with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
 
@@ -3968,7 +4327,7 @@ class basic_json
             return m_value.object->operator[](key);
         }
 
-        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -3987,8 +4346,8 @@ class basic_json
     @pre The element with key @a key must exist. **This precondition is
          enforced with an assertion.**
 
-    @throw std::domain_error if JSON is not an object; example: `"cannot use
-    operator[] with null"`
+    @throw type_error.305 if the JSON value is not an object; in that cases,
+    using the [] operator with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
 
@@ -4011,7 +4370,7 @@ class basic_json
             return m_value.object->find(key)->second;
         }
 
-        JSON_THROW(std::domain_error("cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
     }
 
     /*!
@@ -4024,7 +4383,7 @@ class basic_json
     @code {.cpp}
     try {
         return at(key);
-    } catch(std::out_of_range) {
+    } catch(out_of_range) {
         return default_value;
     }
     @endcode
@@ -4047,8 +4406,8 @@ class basic_json
     @return copy of the element at key @a key or @a default_value if @a key
     is not found
 
-    @throw std::domain_error if JSON is not an object; example: `"cannot use
-    value() with null"`
+    @throw type_error.306 if the JSON value is not an objec; in that cases,
+    using `value()` with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
 
@@ -4080,7 +4439,7 @@ class basic_json
         }
         else
         {
-            JSON_THROW(std::domain_error("cannot use value() with " + type_name()));
+            JSON_THROW(type_error::create(306, "cannot use value() with " + type_name()));
         }
     }
 
@@ -4103,7 +4462,7 @@ class basic_json
     @code {.cpp}
     try {
         return at(ptr);
-    } catch(std::out_of_range) {
+    } catch(out_of_range) {
         return default_value;
     }
     @endcode
@@ -4122,8 +4481,8 @@ class basic_json
     @return copy of the element at key @a key or @a default_value if @a key
     is not found
 
-    @throw std::domain_error if JSON is not an object; example: `"cannot use
-    value() with null"`
+    @throw type_error.306 if the JSON value is not an objec; in that cases,
+    using `value()` with a key makes no sense.
 
     @complexity Logarithmic in the size of the container.
 
@@ -4146,13 +4505,13 @@ class basic_json
             {
                 return ptr.get_checked(this);
             }
-            JSON_CATCH (std::out_of_range&)
+            JSON_CATCH (out_of_range&)
             {
                 return default_value;
             }
         }
 
-        JSON_THROW(std::domain_error("cannot use value() with " + type_name()));
+        JSON_THROW(type_error::create(306, "cannot use value() with " + type_name()));
     }
 
     /*!
@@ -4181,7 +4540,7 @@ class basic_json
     assertions**).
     @post The JSON value remains unchanged.
 
-    @throw std::out_of_range when called on `null` value
+    @throw invalid_iterator.214 when called on `null` value
 
     @liveexample{The following code shows an example for `front()`.,front}
 
@@ -4224,7 +4583,8 @@ class basic_json
     assertions**).
     @post The JSON value remains unchanged.
 
-    @throw std::out_of_range when called on `null` value.
+    @throw invalid_iterator.214 when called on a `null` value. See example
+    below.
 
     @liveexample{The following code shows an example for `back()`.,back}
 
@@ -4268,11 +4628,12 @@ class basic_json
     @post Invalidates iterators and references at or after the point of the
     erase, including the `end()` iterator.
 
-    @throw std::domain_error if called on a `null` value; example: `"cannot
-    use erase() with null"`
-    @throw std::domain_error if called on an iterator which does not belong to
-    the current JSON value; example: `"iterator does not fit current value"`
-    @throw std::out_of_range if called on a primitive type with invalid
+    @throw type_error.307 if called on a `null` value; example: `"cannot use
+    erase() with null"`
+    @throw invalid_iterator.202 if called on an iterator which does not belong
+    to the current JSON value; example: `"iterator does not fit current
+    value"`
+    @throw invalid_iterator.205 if called on a primitive type with invalid
     iterator (i.e., any iterator which is not `begin()`); example: `"iterator
     out of range"`
 
@@ -4303,7 +4664,7 @@ class basic_json
         // make sure iterator fits the current value
         if (this != pos.m_object)
         {
-            JSON_THROW(std::domain_error("iterator does not fit current value"));
+            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
         }
 
         IteratorType result = end();
@@ -4318,7 +4679,7 @@ class basic_json
             {
                 if (not pos.m_it.primitive_iterator.is_begin())
                 {
-                    JSON_THROW(std::out_of_range("iterator out of range"));
+                    JSON_THROW(invalid_iterator::create(205, "iterator out of range"));
                 }
 
                 if (is_string())
@@ -4348,7 +4709,7 @@ class basic_json
 
             default:
             {
-                JSON_THROW(std::domain_error("cannot use erase() with " + type_name()));
+                JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name()));
             }
         }
 
@@ -4375,11 +4736,11 @@ class basic_json
     @post Invalidates iterators and references at or after the point of the
     erase, including the `end()` iterator.
 
-    @throw std::domain_error if called on a `null` value; example: `"cannot
-    use erase() with null"`
-    @throw std::domain_error if called on iterators which does not belong to
-    the current JSON value; example: `"iterators do not fit current value"`
-    @throw std::out_of_range if called on a primitive type with invalid
+    @throw type_error.307 if called on a `null` value; example: `"cannot use
+    erase() with null"`
+    @throw invalid_iterator.203 if called on iterators which does not belong
+    to the current JSON value; example: `"iterators do not fit current value"`
+    @throw invalid_iterator.204 if called on a primitive type with invalid
     iterators (i.e., if `first != begin()` and `last != end()`); example:
     `"iterators out of range"`
 
@@ -4410,7 +4771,7 @@ class basic_json
         // make sure iterator fits the current value
         if (this != first.m_object or this != last.m_object)
         {
-            JSON_THROW(std::domain_error("iterators do not fit current value"));
+            JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value"));
         }
 
         IteratorType result = end();
@@ -4425,7 +4786,7 @@ class basic_json
             {
                 if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end())
                 {
-                    JSON_THROW(std::out_of_range("iterators out of range"));
+                    JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
                 }
 
                 if (is_string())
@@ -4457,7 +4818,7 @@ class basic_json
 
             default:
             {
-                JSON_THROW(std::domain_error("cannot use erase() with " + type_name()));
+                JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name()));
             }
         }
 
@@ -4478,7 +4839,7 @@ class basic_json
     @post References and iterators to the erased elements are invalidated.
     Other references and iterators are not affected.
 
-    @throw std::domain_error when called on a type other than JSON object;
+    @throw type_error.307 when called on a type other than JSON object;
     example: `"cannot use erase() with null"`
 
     @complexity `log(size()) + count(key)`
@@ -4501,7 +4862,7 @@ class basic_json
             return m_value.object->erase(key);
         }
 
-        JSON_THROW(std::domain_error("cannot use erase() with " + type_name()));
+        JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name()));
     }
 
     /*!
@@ -4511,9 +4872,9 @@ class basic_json
 
     @param[in] idx index of the element to remove
 
-    @throw std::domain_error when called on a type other than JSON array;
+    @throw type_error.307 when called on a type other than JSON object;
     example: `"cannot use erase() with null"`
-    @throw std::out_of_range when `idx >= size()`; example: `"array index 17
+    @throw out_of_range.401 when `idx >= size()`; example: `"array index 17
     is out of range"`
 
     @complexity Linear in distance between @a idx and the end of the container.
@@ -4535,14 +4896,14 @@ class basic_json
         {
             if (idx >= size())
             {
-                JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range"));
+                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
             }
 
             m_value.array->erase(m_value.array->begin() + static_cast<difference_type>(idx));
         }
         else
         {
-            JSON_THROW(std::domain_error("cannot use erase() with " + type_name()));
+            JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name()));
         }
     }
 
@@ -4929,6 +5290,8 @@ class basic_json
     reference to the JSON values is returned, so there is no access to the
     underlying iterator.
 
+    @liveexample{The following code shows how the wrapper is used,iterator_wrapper}
+
     @note The name of this function is not yet final and may change in the
     future.
     */
@@ -5244,7 +5607,7 @@ class basic_json
 
     @param[in] val the value to add to the JSON array
 
-    @throw std::domain_error when called on a type other than JSON array or
+    @throw type_error.308 when called on a type other than JSON array or
     null; example: `"cannot use push_back() with number"`
 
     @complexity Amortized constant.
@@ -5260,7 +5623,7 @@ class basic_json
         // push_back only works for null objects or arrays
         if (not(is_null() or is_array()))
         {
-            JSON_THROW(std::domain_error("cannot use push_back() with " + type_name()));
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name()));
         }
 
         // transform null object into an array
@@ -5296,7 +5659,7 @@ class basic_json
         // push_back only works for null objects or arrays
         if (not(is_null() or is_array()))
         {
-            JSON_THROW(std::domain_error("cannot use push_back() with " + type_name()));
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name()));
         }
 
         // transform null object into an array
@@ -5330,7 +5693,7 @@ class basic_json
 
     @param[in] val the value to add to the JSON object
 
-    @throw std::domain_error when called on a type other than JSON object or
+    @throw type_error.308 when called on a type other than JSON object or
     null; example: `"cannot use push_back() with number"`
 
     @complexity Logarithmic in the size of the container, O(log(`size()`)).
@@ -5346,7 +5709,7 @@ class basic_json
         // push_back only works for null objects or objects
         if (not(is_null() or is_object()))
         {
-            JSON_THROW(std::domain_error("cannot use push_back() with " + type_name()));
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name()));
         }
 
         // transform null object into an object
@@ -5384,7 +5747,7 @@ class basic_json
     @ref push_back(const typename object_t::value_type&). Otherwise, @a init
     is converted to a JSON value and added using @ref push_back(basic_json&&).
 
-    @param init  an initializer list
+    @param[in] init  an initializer list
 
     @complexity Linear in the size of the initializer list @a init.
 
@@ -5429,7 +5792,7 @@ class basic_json
     @param[in] args arguments to forward to a constructor of @ref basic_json
     @tparam Args compatible types to create a @ref basic_json object
 
-    @throw std::domain_error when called on a type other than JSON array or
+    @throw type_error.311 when called on a type other than JSON array or
     null; example: `"cannot use emplace_back() with number"`
 
     @complexity Amortized constant.
@@ -5446,7 +5809,7 @@ class basic_json
         // emplace_back only works for null objects or arrays
         if (not(is_null() or is_array()))
         {
-            JSON_THROW(std::domain_error("cannot use emplace_back() with " + type_name()));
+            JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + type_name()));
         }
 
         // transform null object into an array
@@ -5476,7 +5839,7 @@ class basic_json
             already-existing element if no insertion happened, and a bool
             denoting whether the insertion took place.
 
-    @throw std::domain_error when called on a type other than JSON object or
+    @throw type_error.311 when called on a type other than JSON object or
     null; example: `"cannot use emplace() with number"`
 
     @complexity Logarithmic in the size of the container, O(log(`size()`)).
@@ -5494,7 +5857,7 @@ class basic_json
         // emplace only works for null objects or arrays
         if (not(is_null() or is_object()))
         {
-            JSON_THROW(std::domain_error("cannot use emplace() with " + type_name()));
+            JSON_THROW(type_error::create(311, "cannot use emplace() with " + type_name()));
         }
 
         // transform null object into an object
@@ -5525,10 +5888,10 @@ class basic_json
     @param[in] val element to insert
     @return iterator pointing to the inserted @a val.
 
-    @throw std::domain_error if called on JSON values other than arrays;
+    @throw type_error.309 if called on JSON values other than arrays;
     example: `"cannot use insert() with string"`
-    @throw std::domain_error if @a pos is not an iterator of *this; example:
-    `"iterator does not fit current value"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
 
     @complexity Constant plus linear in the distance between @a pos and end of
     the container.
@@ -5545,7 +5908,7 @@ class basic_json
             // check if iterator pos fits to this JSON value
             if (pos.m_object != this)
             {
-                JSON_THROW(std::domain_error("iterator does not fit current value"));
+                JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
             }
 
             // insert to array and return iterator
@@ -5554,7 +5917,7 @@ class basic_json
             return result;
         }
 
-        JSON_THROW(std::domain_error("cannot use insert() with " + type_name()));
+        JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
     }
 
     /*!
@@ -5578,10 +5941,10 @@ class basic_json
     @return iterator pointing to the first element inserted, or @a pos if
     `cnt==0`
 
-    @throw std::domain_error if called on JSON values other than arrays;
-    example: `"cannot use insert() with string"`
-    @throw std::domain_error if @a pos is not an iterator of *this; example:
-    `"iterator does not fit current value"`
+    @throw type_error.309 if called on JSON values other than arrays; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
 
     @complexity Linear in @a cnt plus linear in the distance between @a pos
     and end of the container.
@@ -5598,7 +5961,7 @@ class basic_json
             // check if iterator pos fits to this JSON value
             if (pos.m_object != this)
             {
-                JSON_THROW(std::domain_error("iterator does not fit current value"));
+                JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
             }
 
             // insert to array and return iterator
@@ -5607,7 +5970,7 @@ class basic_json
             return result;
         }
 
-        JSON_THROW(std::domain_error("cannot use insert() with " + type_name()));
+        JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
     }
 
     /*!
@@ -5620,13 +5983,13 @@ class basic_json
     @param[in] first begin of the range of elements to insert
     @param[in] last end of the range of elements to insert
 
-    @throw std::domain_error if called on JSON values other than arrays;
-    example: `"cannot use insert() with string"`
-    @throw std::domain_error if @a pos is not an iterator of *this; example:
-    `"iterator does not fit current value"`
-    @throw std::domain_error if @a first and @a last do not belong to the same
-    JSON value; example: `"iterators do not fit"`
-    @throw std::domain_error if @a first or @a last are iterators into
+    @throw type_error.309 if called on JSON values other than arrays; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
+    @throw invalid_iterator.210 if @a first and @a last do not belong to the
+    same JSON value; example: `"iterators do not fit"`
+    @throw invalid_iterator.211 if @a first or @a last are iterators into
     container for which insert is called; example: `"passed iterators may not
     belong to container"`
 
@@ -5645,24 +6008,24 @@ class basic_json
         // insert only works for arrays
         if (not is_array())
         {
-            JSON_THROW(std::domain_error("cannot use insert() with " + type_name()));
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
         }
 
         // check if iterator pos fits to this JSON value
         if (pos.m_object != this)
         {
-            JSON_THROW(std::domain_error("iterator does not fit current value"));
+            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
         }
 
         // check if range iterators belong to the same JSON object
         if (first.m_object != last.m_object)
         {
-            JSON_THROW(std::domain_error("iterators do not fit"));
+            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
         }
 
         if (first.m_object == this or last.m_object == this)
         {
-            JSON_THROW(std::domain_error("passed iterators may not belong to container"));
+            JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container"));
         }
 
         // insert to array and return iterator
@@ -5683,10 +6046,10 @@ class basic_json
     the end() iterator
     @param[in] ilist initializer list to insert the values from
 
-    @throw std::domain_error if called on JSON values other than arrays;
-    example: `"cannot use insert() with string"`
-    @throw std::domain_error if @a pos is not an iterator of *this; example:
-    `"iterator does not fit current value"`
+    @throw type_error.309 if called on JSON values other than arrays; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if @a pos is not an iterator of *this;
+    example: `"iterator does not fit current value"`
 
     @return iterator pointing to the first element inserted, or @a pos if
     `ilist` is empty
@@ -5703,13 +6066,13 @@ class basic_json
         // insert only works for arrays
         if (not is_array())
         {
-            JSON_THROW(std::domain_error("cannot use insert() with " + type_name()));
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
         }
 
         // check if iterator pos fits to this JSON value
         if (pos.m_object != this)
         {
-            JSON_THROW(std::domain_error("iterator does not fit current value"));
+            JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
         }
 
         // insert to array and return iterator
@@ -5718,6 +6081,52 @@ class basic_json
         return result;
     }
 
+    /*!
+    @brief inserts elements
+
+    Inserts elements from range `[first, last)`.
+
+    @param[in] first begin of the range of elements to insert
+    @param[in] last end of the range of elements to insert
+
+    @throw type_error.309 if called on JSON values other than objects; example:
+    `"cannot use insert() with string"`
+    @throw invalid_iterator.202 if iterator @a first or @a last does does not
+    point to an object; example: `"iterators first and last must point to
+    objects"`
+    @throw invalid_iterator.210 if @a first and @a last do not belong to the
+    same JSON value; example: `"iterators do not fit"`
+
+    @complexity Logarithmic: `O(N*log(size() + N))`, where `N` is the number
+    of elements to insert.
+
+    @liveexample{The example shows how `insert()` is used.,insert__range_object}
+
+    @since version 3.0.0
+    */
+    void insert(const_iterator first, const_iterator last)
+    {
+        // insert only works for objects
+        if (not is_object())
+        {
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
+        }
+
+        // check if range iterators belong to the same JSON object
+        if (first.m_object != last.m_object)
+        {
+            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
+        }
+
+        // passed iterators must belong to objects
+        if (not first.m_object->is_object() or not first.m_object->is_object())
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
+        }
+
+        m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator);
+    }
+
     /*!
     @brief exchanges the values
 
@@ -5757,8 +6166,8 @@ class basic_json
 
     @param[in,out] other array to exchange the contents with
 
-    @throw std::domain_error when JSON value is not an array; example:
-    `"cannot use swap() with string"`
+    @throw type_error.310 when JSON value is not an array; example: `"cannot
+    use swap() with string"`
 
     @complexity Constant.
 
@@ -5776,7 +6185,7 @@ class basic_json
         }
         else
         {
-            JSON_THROW(std::domain_error("cannot use swap() with " + type_name()));
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name()));
         }
     }
 
@@ -5790,7 +6199,7 @@ class basic_json
 
     @param[in,out] other object to exchange the contents with
 
-    @throw std::domain_error when JSON value is not an object; example:
+    @throw type_error.310 when JSON value is not an object; example:
     `"cannot use swap() with string"`
 
     @complexity Constant.
@@ -5809,7 +6218,7 @@ class basic_json
         }
         else
         {
-            JSON_THROW(std::domain_error("cannot use swap() with " + type_name()));
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name()));
         }
     }
 
@@ -5823,7 +6232,7 @@ class basic_json
 
     @param[in,out] other string to exchange the contents with
 
-    @throw std::domain_error when JSON value is not a string; example: `"cannot
+    @throw type_error.310 when JSON value is not a string; example: `"cannot
     use swap() with boolean"`
 
     @complexity Constant.
@@ -5842,7 +6251,7 @@ class basic_json
         }
         else
         {
-            JSON_THROW(std::domain_error("cannot use swap() with " + type_name()));
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name()));
         }
     }
 
@@ -5861,13 +6270,17 @@ class basic_json
 
     Compares two JSON values for equality according to the following rules:
     - Two JSON values are equal if (1) they are from the same type and (2)
-      their stored values are the same.
+      their stored values are the same according to their respective
+      `operator==`.
     - Integer and floating-point numbers are automatically converted before
       comparison. Floating-point numbers are compared indirectly: two
       floating-point numbers `f1` and `f2` are considered equal if neither
-      `f1 > f2` nor `f2 > f1` holds.
+      `f1 > f2` nor `f2 > f1` holds. Note than two NaN values are always
+      treated as unequal.
     - Two JSON null values are equal.
 
+    @note NaN values never compare equal to themselves or to other NaN values.
+
     @param[in] lhs  first JSON value to consider
     @param[in] rhs  second JSON value to consider
     @return whether the values @a lhs and @a rhs are equal
@@ -6054,7 +6467,7 @@ class basic_json
             {
                 case value_t::array:
                 {
-                    return *lhs.m_value.array < *rhs.m_value.array;
+                    return (*lhs.m_value.array) < (*rhs.m_value.array);
                 }
                 case value_t::object:
                 {
@@ -6121,6 +6534,28 @@ class basic_json
         return operator<(lhs_type, rhs_type);
     }
 
+    /*!
+    @brief comparison: less than
+    @copydoc operator<(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs < basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: less than
+    @copydoc operator<(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) < rhs);
+    }
+
     /*!
     @brief comparison: less than or equal
 
@@ -6143,6 +6578,28 @@ class basic_json
         return not (rhs < lhs);
     }
 
+    /*!
+    @brief comparison: less than or equal
+    @copydoc operator<=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs <= basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: less than or equal
+    @copydoc operator<=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) <= rhs);
+    }
+
     /*!
     @brief comparison: greater than
 
@@ -6165,6 +6622,28 @@ class basic_json
         return not (lhs <= rhs);
     }
 
+    /*!
+    @brief comparison: greater than
+    @copydoc operator>(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs > basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: greater than
+    @copydoc operator>(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) > rhs);
+    }
+
     /*!
     @brief comparison: greater than or equal
 
@@ -6187,8 +6666,133 @@ class basic_json
         return not (lhs < rhs);
     }
 
+    /*!
+    @brief comparison: greater than or equal
+    @copydoc operator>=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept
+    {
+        return (lhs >= basic_json(rhs));
+    }
+
+    /*!
+    @brief comparison: greater than or equal
+    @copydoc operator>=(const_reference, const_reference)
+    */
+    template<typename ScalarType, typename std::enable_if<
+                 std::is_scalar<ScalarType>::value, int>::type = 0>
+    friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept
+    {
+        return (basic_json(lhs) >= rhs);
+    }
+
     /// @}
 
+  private:
+    /////////////////////
+    // output adapters //
+    /////////////////////
+
+    /// abstract output adapter interface
+    template<typename CharType>
+    class output_adapter
+    {
+      public:
+        virtual void write_character(CharType c) = 0;
+        virtual void write_characters(const CharType* s, size_t length) = 0;
+        virtual ~output_adapter() {}
+
+        static std::shared_ptr<output_adapter<CharType>> create(std::vector<CharType>& vec)
+        {
+            return std::shared_ptr<output_adapter>(new output_vector_adapter<CharType>(vec));
+        }
+
+        static std::shared_ptr<output_adapter<CharType>> create(std::ostream& s)
+        {
+            return std::shared_ptr<output_adapter>(new output_stream_adapter<CharType>(s));
+        }
+
+        static std::shared_ptr<output_adapter<CharType>> create(std::string& s)
+        {
+            return std::shared_ptr<output_adapter>(new output_string_adapter<CharType>(s));
+        }
+    };
+
+    /// a type to simplify interfaces
+    template<typename CharType>
+    using output_adapter_t = std::shared_ptr<output_adapter<CharType>>;
+
+    /// output adapter for byte vectors
+    template<typename CharType>
+    class output_vector_adapter : public output_adapter<CharType>
+    {
+      public:
+        output_vector_adapter(std::vector<CharType>& vec)
+            : v(vec)
+        {}
+
+        void write_character(CharType c) override
+        {
+            v.push_back(c);
+        }
+
+        void write_characters(const CharType* s, size_t length) override
+        {
+            std::copy(s, s + length, std::back_inserter(v));
+        }
+
+      private:
+        std::vector<CharType>& v;
+    };
+
+    /// putput adatpter for output streams
+    template<typename CharType>
+    class output_stream_adapter : public output_adapter<CharType>
+    {
+      public:
+        output_stream_adapter(std::basic_ostream<CharType>& s)
+            : stream(s)
+        {}
+
+        void write_character(CharType c) override
+        {
+            stream.put(c);
+        }
+
+        void write_characters(const CharType* s, size_t length) override
+        {
+            stream.write(s, static_cast<std::streamsize>(length));
+        }
+
+      private:
+        std::basic_ostream<CharType>& stream;
+    };
+
+    /// output adapter for basic_string
+    template<typename CharType>
+    class output_string_adapter : public output_adapter<CharType>
+    {
+      public:
+        output_string_adapter(std::string& s)
+            : str(s)
+        {}
+
+        void write_character(CharType c) override
+        {
+            str.push_back(c);
+        }
+
+        void write_characters(const CharType* s, size_t length) override
+        {
+            str.append(s, length);
+        }
+
+      private:
+        std::basic_string<CharType>& str;
+    };
+
 
     ///////////////////
     // serialization //
@@ -6197,15 +6801,613 @@ class basic_json
     /// @name serialization
     /// @{
 
+  private:
+    /*!
+    @brief wrapper around the serialization functions
+    */
+    class serializer
+    {
+      public:
+        /*!
+        @param[in] s  output stream to serialize to
+        @param[in] ichar  indentation character to use
+        */
+        serializer(output_adapter_t<char> s, const char ichar)
+            : o(s), loc(std::localeconv()),
+              thousands_sep(!loc->thousands_sep ? '\0' : loc->thousands_sep[0]),
+              decimal_point(!loc->decimal_point ? '\0' : loc->decimal_point[0]),
+              indent_char(ichar), indent_string(512, indent_char)
+        {}
+
+        // delete because of pointer members
+        serializer(const serializer&) = delete;
+        serializer& operator=(const serializer&) = delete;
+
+        /*!
+        @brief internal implementation of the serialization function
+
+        This function is called by the public member function dump and
+        organizes the serialization internally. The indentation level is
+        propagated as additional parameter. In case of arrays and objects, the
+        function is called recursively.
+
+        - strings and object keys are escaped using `escape_string()`
+        - integer numbers are converted implicitly via `operator<<`
+        - floating-point numbers are converted to a string using `"%g"` format
+
+        @param[in] val             value to serialize
+        @param[in] pretty_print    whether the output shall be pretty-printed
+        @param[in] indent_step     the indent level
+        @param[in] current_indent  the current indent level (only used internally)
+        */
+        void dump(const basic_json& val,
+                  const bool pretty_print,
+                  const unsigned int indent_step,
+                  const unsigned int current_indent = 0)
+        {
+            switch (val.m_type)
+            {
+                case value_t::object:
+                {
+                    if (val.m_value.object->empty())
+                    {
+                        o->write_characters("{}", 2);
+                        return;
+                    }
+
+                    if (pretty_print)
+                    {
+                        o->write_characters("{\n", 2);
+
+                        // variable to hold indentation for recursive calls
+                        const auto new_indent = current_indent + indent_step;
+                        if (indent_string.size() < new_indent)
+                        {
+                            indent_string.resize(new_indent, ' ');
+                        }
+
+                        // first n-1 elements
+                        auto i = val.m_value.object->cbegin();
+                        for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
+                        {
+                            o->write_characters(indent_string.c_str(), new_indent);
+                            o->write_character('\"');
+                            dump_escaped(i->first);
+                            o->write_characters("\": ", 3);
+                            dump(i->second, true, indent_step, new_indent);
+                            o->write_characters(",\n", 2);
+                        }
+
+                        // last element
+                        assert(i != val.m_value.object->cend());
+                        o->write_characters(indent_string.c_str(), new_indent);
+                        o->write_character('\"');
+                        dump_escaped(i->first);
+                        o->write_characters("\": ", 3);
+                        dump(i->second, true, indent_step, new_indent);
+
+                        o->write_character('\n');
+                        o->write_characters(indent_string.c_str(), current_indent);
+                        o->write_character('}');
+                    }
+                    else
+                    {
+                        o->write_character('{');
+
+                        // first n-1 elements
+                        auto i = val.m_value.object->cbegin();
+                        for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
+                        {
+                            o->write_character('\"');
+                            dump_escaped(i->first);
+                            o->write_characters("\":", 2);
+                            dump(i->second, false, indent_step, current_indent);
+                            o->write_character(',');
+                        }
+
+                        // last element
+                        assert(i != val.m_value.object->cend());
+                        o->write_character('\"');
+                        dump_escaped(i->first);
+                        o->write_characters("\":", 2);
+                        dump(i->second, false, indent_step, current_indent);
+
+                        o->write_character('}');
+                    }
+
+                    return;
+                }
+
+                case value_t::array:
+                {
+                    if (val.m_value.array->empty())
+                    {
+                        o->write_characters("[]", 2);
+                        return;
+                    }
+
+                    if (pretty_print)
+                    {
+                        o->write_characters("[\n", 2);
+
+                        // variable to hold indentation for recursive calls
+                        const auto new_indent = current_indent + indent_step;
+                        if (indent_string.size() < new_indent)
+                        {
+                            indent_string.resize(new_indent, ' ');
+                        }
+
+                        // first n-1 elements
+                        for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i)
+                        {
+                            o->write_characters(indent_string.c_str(), new_indent);
+                            dump(*i, true, indent_step, new_indent);
+                            o->write_characters(",\n", 2);
+                        }
+
+                        // last element
+                        assert(not val.m_value.array->empty());
+                        o->write_characters(indent_string.c_str(), new_indent);
+                        dump(val.m_value.array->back(), true, indent_step, new_indent);
+
+                        o->write_character('\n');
+                        o->write_characters(indent_string.c_str(), current_indent);
+                        o->write_character(']');
+                    }
+                    else
+                    {
+                        o->write_character('[');
+
+                        // first n-1 elements
+                        for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i)
+                        {
+                            dump(*i, false, indent_step, current_indent);
+                            o->write_character(',');
+                        }
+
+                        // last element
+                        assert(not val.m_value.array->empty());
+                        dump(val.m_value.array->back(), false, indent_step, current_indent);
+
+                        o->write_character(']');
+                    }
+
+                    return;
+                }
+
+                case value_t::string:
+                {
+                    o->write_character('\"');
+                    dump_escaped(*val.m_value.string);
+                    o->write_character('\"');
+                    return;
+                }
+
+                case value_t::boolean:
+                {
+                    if (val.m_value.boolean)
+                    {
+                        o->write_characters("true", 4);
+                    }
+                    else
+                    {
+                        o->write_characters("false", 5);
+                    }
+                    return;
+                }
+
+                case value_t::number_integer:
+                {
+                    dump_integer(val.m_value.number_integer);
+                    return;
+                }
+
+                case value_t::number_unsigned:
+                {
+                    dump_integer(val.m_value.number_unsigned);
+                    return;
+                }
+
+                case value_t::number_float:
+                {
+                    dump_float(val.m_value.number_float);
+                    return;
+                }
+
+                case value_t::discarded:
+                {
+                    o->write_characters("<discarded>", 11);
+                    return;
+                }
+
+                case value_t::null:
+                {
+                    o->write_characters("null", 4);
+                    return;
+                }
+            }
+        }
+
+      private:
+        /*!
+        @brief calculates the extra space to escape a JSON string
+
+        @param[in] s  the string to escape
+        @return the number of characters required to escape string @a s
+
+        @complexity Linear in the length of string @a s.
+        */
+        static std::size_t extra_space(const string_t& s) noexcept
+        {
+            return std::accumulate(s.begin(), s.end(), size_t{},
+                                   [](size_t res, typename string_t::value_type c)
+            {
+                switch (c)
+                {
+                    case '"':
+                    case '\\':
+                    case '\b':
+                    case '\f':
+                    case '\n':
+                    case '\r':
+                    case '\t':
+                    {
+                        // from c (1 byte) to \x (2 bytes)
+                        return res + 1;
+                    }
+
+                    case 0x00:
+                    case 0x01:
+                    case 0x02:
+                    case 0x03:
+                    case 0x04:
+                    case 0x05:
+                    case 0x06:
+                    case 0x07:
+                    case 0x0b:
+                    case 0x0e:
+                    case 0x0f:
+                    case 0x10:
+                    case 0x11:
+                    case 0x12:
+                    case 0x13:
+                    case 0x14:
+                    case 0x15:
+                    case 0x16:
+                    case 0x17:
+                    case 0x18:
+                    case 0x19:
+                    case 0x1a:
+                    case 0x1b:
+                    case 0x1c:
+                    case 0x1d:
+                    case 0x1e:
+                    case 0x1f:
+                    {
+                        // from c (1 byte) to \uxxxx (6 bytes)
+                        return res + 5;
+                    }
+
+                    default:
+                    {
+                        return res;
+                    }
+                }
+            });
+        }
+
+        /*!
+        @brief dump escaped string
+
+        Escape a string by replacing certain special characters by a sequence
+        of an escape character (backslash) and another character and other
+        control characters by a sequence of "\u" followed by a four-digit hex
+        representation. The escaped string is written to output stream @a o.
+
+        @param[in] s  the string to escape
+
+        @complexity Linear in the length of string @a s.
+        */
+        void dump_escaped(const string_t& s) const
+        {
+            const auto space = extra_space(s);
+            if (space == 0)
+            {
+                o->write_characters(s.c_str(), s.size());
+                return;
+            }
+
+            // create a result string of necessary size
+            string_t result(s.size() + space, '\\');
+            std::size_t pos = 0;
+
+            for (const auto& c : s)
+            {
+                switch (c)
+                {
+                    // quotation mark (0x22)
+                    case '"':
+                    {
+                        result[pos + 1] = '"';
+                        pos += 2;
+                        break;
+                    }
+
+                    // reverse solidus (0x5c)
+                    case '\\':
+                    {
+                        // nothing to change
+                        pos += 2;
+                        break;
+                    }
+
+                    // backspace (0x08)
+                    case '\b':
+                    {
+                        result[pos + 1] = 'b';
+                        pos += 2;
+                        break;
+                    }
+
+                    // formfeed (0x0c)
+                    case '\f':
+                    {
+                        result[pos + 1] = 'f';
+                        pos += 2;
+                        break;
+                    }
+
+                    // newline (0x0a)
+                    case '\n':
+                    {
+                        result[pos + 1] = 'n';
+                        pos += 2;
+                        break;
+                    }
+
+                    // carriage return (0x0d)
+                    case '\r':
+                    {
+                        result[pos + 1] = 'r';
+                        pos += 2;
+                        break;
+                    }
+
+                    // horizontal tab (0x09)
+                    case '\t':
+                    {
+                        result[pos + 1] = 't';
+                        pos += 2;
+                        break;
+                    }
+
+                    case 0x00:
+                    case 0x01:
+                    case 0x02:
+                    case 0x03:
+                    case 0x04:
+                    case 0x05:
+                    case 0x06:
+                    case 0x07:
+                    case 0x0b:
+                    case 0x0e:
+                    case 0x0f:
+                    case 0x10:
+                    case 0x11:
+                    case 0x12:
+                    case 0x13:
+                    case 0x14:
+                    case 0x15:
+                    case 0x16:
+                    case 0x17:
+                    case 0x18:
+                    case 0x19:
+                    case 0x1a:
+                    case 0x1b:
+                    case 0x1c:
+                    case 0x1d:
+                    case 0x1e:
+                    case 0x1f:
+                    {
+                        // convert a number 0..15 to its hex representation
+                        // (0..f)
+                        static const char hexify[16] =
+                        {
+                            '0', '1', '2', '3', '4', '5', '6', '7',
+                            '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
+                        };
+
+                        // print character c as \uxxxx
+                        for (const char m :
+                    { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f]
+                        })
+                        {
+                            result[++pos] = m;
+                        }
+
+                        ++pos;
+                        break;
+                    }
+
+                    default:
+                    {
+                        // all other characters are added as-is
+                        result[pos++] = c;
+                        break;
+                    }
+                }
+            }
+
+            assert(pos == s.size() + space);
+            o->write_characters(result.c_str(), result.size());
+        }
+
+        /*!
+        @brief dump an integer
+
+        Dump a given integer to output stream @a o. Works internally with
+        @a number_buffer.
+
+        @param[in] x  integer number (signed or unsigned) to dump
+        @tparam NumberType either @a number_integer_t or @a number_unsigned_t
+        */
+        template<typename NumberType, detail::enable_if_t <
+                     std::is_same<NumberType, number_unsigned_t>::value or
+                     std::is_same<NumberType, number_integer_t>::value, int> = 0>
+        void dump_integer(NumberType x)
+        {
+            // special case for "0"
+            if (x == 0)
+            {
+                o->write_character('0');
+                return;
+            }
+
+            const bool is_negative = x < 0;
+            size_t i = 0;
+
+            // spare 1 byte for '\0'
+            while (x != 0 and i < number_buffer.size() - 1)
+            {
+                const auto digit = std::labs(static_cast<long>(x % 10));
+                number_buffer[i++] = static_cast<char>('0' + digit);
+                x /= 10;
+            }
+
+            // make sure the number has been processed completely
+            assert(x == 0);
+
+            if (is_negative)
+            {
+                // make sure there is capacity for the '-'
+                assert(i < number_buffer.size() - 2);
+                number_buffer[i++] = '-';
+            }
+
+            std::reverse(number_buffer.begin(), number_buffer.begin() + i);
+            o->write_characters(number_buffer.data(), i);
+        }
+
+        /*!
+        @brief dump a floating-point number
+
+        Dump a given floating-point number to output stream @a o. Works
+        internally with @a number_buffer.
+
+        @param[in] x  floating-point number to dump
+        */
+        void dump_float(number_float_t x)
+        {
+            // NaN / inf
+            if (not std::isfinite(x) or std::isnan(x))
+            {
+                o->write_characters("null", 4);
+                return;
+            }
+
+            // special case for 0.0 and -0.0
+            if (x == 0)
+            {
+                if (std::signbit(x))
+                {
+                    o->write_characters("-0.0", 4);
+                }
+                else
+                {
+                    o->write_characters("0.0", 3);
+                }
+                return;
+            }
+
+            // get number of digits for a text -> float -> text round-trip
+            static constexpr auto d = std::numeric_limits<number_float_t>::digits10;
+
+            // the actual conversion
+            std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(),
+                                          "%.*g", d, x);
+
+            // negative value indicates an error
+            assert(len > 0);
+            // check if buffer was large enough
+            assert(static_cast<size_t>(len) < number_buffer.size());
+
+            // erase thousands separator
+            if (thousands_sep != '\0')
+            {
+                const auto end = std::remove(number_buffer.begin(),
+                                             number_buffer.begin() + len,
+                                             thousands_sep);
+                std::fill(end, number_buffer.end(), '\0');
+                assert((end - number_buffer.begin()) <= len);
+                len = (end - number_buffer.begin());
+            }
+
+            // convert decimal point to '.'
+            if (decimal_point != '\0' and decimal_point != '.')
+            {
+                for (auto& c : number_buffer)
+                {
+                    if (c == decimal_point)
+                    {
+                        c = '.';
+                        break;
+                    }
+                }
+            }
+
+            o->write_characters(number_buffer.data(), static_cast<size_t>(len));
+
+            // determine if need to append ".0"
+            const bool value_is_int_like = std::none_of(number_buffer.begin(),
+                                           number_buffer.begin() + len + 1,
+                                           [](char c)
+            {
+                return c == '.' or c == 'e';
+            });
+
+            if (value_is_int_like)
+            {
+                o->write_characters(".0", 2);
+            }
+        }
+
+      private:
+        /// the output of the serializer
+        output_adapter_t<char> o = nullptr;
+
+        /// a (hopefully) large enough character buffer
+        std::array<char, 64> number_buffer{{}};
+
+        /// the locale
+        const std::lconv* loc = nullptr;
+        /// the locale's thousand separator character
+        const char thousands_sep = '\0';
+        /// the locale's decimal point character
+        const char decimal_point = '\0';
+
+        /// the indentation character
+        const char indent_char;
+
+        /// the indentation string
+        string_t indent_string;
+    };
+
+  public:
     /*!
     @brief serialize to stream
 
     Serialize the given JSON value @a j to the output stream @a o. The JSON
-    value will be serialized using the @ref dump member function. The
-    indentation of the output can be controlled with the member variable
-    `width` of the output stream @a o. For instance, using the manipulator
-    `std::setw(4)` on @a o sets the indentation level to `4` and the
-    serialization result is the same as calling `dump(4)`.
+    value will be serialized using the @ref dump member function.
+
+    - The indentation of the output can be controlled with the member variable
+      `width` of the output stream @a o. For instance, using the manipulator
+      `std::setw(4)` on @a o sets the indentation level to `4` and the
+      serialization result is the same as calling `dump(4)`.
+
+    - The indentation characrer can be controlled with the member variable
+      `fill` of the output stream @a o. For instance, the manipulator
+      `std::setfill('\\t')` sets indentation to use a tab character rather than
+      the default space character.
 
     @param[in,out] o  stream to serialize to
     @param[in] j  JSON value to serialize
@@ -6217,7 +7419,7 @@ class basic_json
     @liveexample{The example below shows the serialization with different
     parameters to `width` to adjust the indentation level.,operator_serialize}
 
-    @since version 1.0.0
+    @since version 1.0.0; indentaction character added in version 3.0.0
     */
     friend std::ostream& operator<<(std::ostream& o, const basic_json& j)
     {
@@ -6229,15 +7431,19 @@ class basic_json
         o.width(0);
 
         // do the actual serialization
-        j.dump(o, pretty_print, static_cast<unsigned int>(indentation));
-
+        serializer s(output_adapter<char>::create(o), o.fill());
+        s.dump(j, pretty_print, static_cast<unsigned int>(indentation));
         return o;
     }
 
     /*!
     @brief serialize to stream
-    @copydoc operator<<(std::ostream&, const basic_json&)
+    @deprecated This stream operator is deprecated and will be removed in a
+                future version of the library. Please use
+                @ref std::ostream& operator<<(std::ostream&, const basic_json&)
+                instead; that is, replace calls like `j >> o;` with `o << j;`.
     */
+    JSON_DEPRECATED
     friend std::ostream& operator>>(const basic_json& j, std::ostream& o)
     {
         return o << j;
@@ -6269,6 +7475,11 @@ class basic_json
 
     @return result of the deserialization
 
+    @throw parse_error.101 if a parse error occurs; example: `""unexpected end
+    of input; expected string literal""`
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
     @complexity Linear in the length of the input. The parser is a predictive
     LL(1) parser. The complexity can be higher if the parser callback function
     @a cb has a super-linear complexity.
@@ -6288,6 +7499,13 @@ class basic_json
         return parse(std::begin(array), std::end(array), cb);
     }
 
+    template<class T, std::size_t N>
+    static bool accept(T (&array)[N])
+    {
+        // delegate the call to the iterator-range accept overload
+        return accept(std::begin(array), std::end(array));
+    }
+
     /*!
     @brief deserialize from string literal
 
@@ -6299,6 +7517,10 @@ class basic_json
 
     @return result of the deserialization
 
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
     @complexity Linear in the length of the input. The parser is a predictive
     LL(1) parser. The complexity can be higher if the parser callback function
     @a cb has a super-linear complexity.
@@ -6322,7 +7544,16 @@ class basic_json
     static basic_json parse(const CharT s,
                             const parser_callback_t cb = nullptr)
     {
-        return parser(reinterpret_cast<const char*>(s), cb).parse();
+        return parser(input_adapter::create(s), cb).parse(true);
+    }
+
+    template<typename CharT, typename std::enable_if<
+                 std::is_pointer<CharT>::value and
+                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
+                 sizeof(typename std::remove_pointer<CharT>::type) == 1, int>::type = 0>
+    static bool accept(const CharT s)
+    {
+        return parser(input_adapter::create(s)).accept(true);
     }
 
     /*!
@@ -6335,6 +7566,11 @@ class basic_json
 
     @return result of the deserialization
 
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+    @throw parse_error.111 if input stream is in a bad state
+
     @complexity Linear in the length of the input. The parser is a predictive
     LL(1) parser. The complexity can be higher if the parser callback function
     @a cb has a super-linear complexity.
@@ -6352,7 +7588,12 @@ class basic_json
     static basic_json parse(std::istream& i,
                             const parser_callback_t cb = nullptr)
     {
-        return parser(i, cb).parse();
+        return parser(input_adapter::create(i), cb).parse(true);
+    }
+
+    static bool accept(std::istream& i)
+    {
+        return parser(input_adapter::create(i)).accept(true);
     }
 
     /*!
@@ -6361,7 +7602,12 @@ class basic_json
     static basic_json parse(std::istream&& i,
                             const parser_callback_t cb = nullptr)
     {
-        return parser(i, cb).parse();
+        return parser(input_adapter::create(i), cb).parse(true);
+    }
+
+    static bool accept(std::istream&& i)
+    {
+        return parser(input_adapter::create(i)).accept(true);
     }
 
     /*!
@@ -6394,6 +7640,10 @@ class basic_json
 
     @return result of the deserialization
 
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
     @complexity Linear in the length of the input. The parser is a predictive
     LL(1) parser. The complexity can be higher if the parser callback function
     @a cb has a super-linear complexity.
@@ -6412,27 +7662,16 @@ class basic_json
     static basic_json parse(IteratorType first, IteratorType last,
                             const parser_callback_t cb = nullptr)
     {
-        // assertion to check that the iterator range is indeed contiguous,
-        // see http://stackoverflow.com/a/35008842/266378 for more discussion
-        assert(std::accumulate(first, last, std::pair<bool, int>(true, 0),
-                               [&first](std::pair<bool, int> res, decltype(*first) val)
-        {
-            res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
-            return res;
-        }).first);
+        return parser(input_adapter::create(first, last), cb).parse(true);
+    }
 
-        // assertion to check that each element is 1 byte long
-        static_assert(sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
-                      "each element in the iterator range must have the size of 1 byte");
-
-        // if iterator range is empty, create a parser with an empty string
-        // to generate "unexpected EOF" error message
-        if (std::distance(first, last) <= 0)
-        {
-            return parser("").parse();
-        }
-
-        return parser(first, last, cb).parse();
+    template<class IteratorType, typename std::enable_if<
+                 std::is_base_of<
+                     std::random_access_iterator_tag,
+                     typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
+    static bool accept(IteratorType first, IteratorType last)
+    {
+        return parser(input_adapter::create(first, last)).accept(true);
     }
 
     /*!
@@ -6464,6 +7703,10 @@ class basic_json
 
     @return result of the deserialization
 
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+
     @complexity Linear in the length of the input. The parser is a predictive
     LL(1) parser. The complexity can be higher if the parser callback function
     @a cb has a super-linear complexity.
@@ -6488,6 +7731,32 @@ class basic_json
         return parse(std::begin(c), std::end(c), cb);
     }
 
+    template<class ContiguousContainer, typename std::enable_if<
+                 not std::is_pointer<ContiguousContainer>::value and
+                 std::is_base_of<
+                     std::random_access_iterator_tag,
+                     typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value
+                 , int>::type = 0>
+    static bool accept(const ContiguousContainer& c)
+    {
+        // delegate the call to the iterator-range accept overload
+        return accept(std::begin(c), std::end(c));
+    }
+
+    /*!
+    @brief deserialize from stream
+    @deprecated This stream operator is deprecated and will be removed in a
+                future version of the library. Please use
+                @ref std::istream& operator>>(std::istream&, basic_json&)
+                instead; that is, replace calls like `j << i;` with `i >> j;`.
+    */
+    JSON_DEPRECATED
+    friend std::istream& operator<<(basic_json& j, std::istream& i)
+    {
+        j = parser(input_adapter::create(i)).parse(false);
+        return i;
+    }
+
     /*!
     @brief deserialize from stream
 
@@ -6496,7 +7765,10 @@ class basic_json
     @param[in,out] i  input stream to read a serialized JSON value from
     @param[in,out] j  JSON value to write the deserialized input to
 
-    @throw std::invalid_argument in case of parse errors
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+    @throw parse_error.111 if input stream is in a bad state
 
     @complexity Linear in the length of the input. The parser is a predictive
     LL(1) parser.
@@ -6511,1538 +7783,14 @@ class basic_json
 
     @since version 1.0.0
     */
-    friend std::istream& operator<<(basic_json& j, std::istream& i)
-    {
-        j = parser(i).parse();
-        return i;
-    }
-
-    /*!
-    @brief deserialize from stream
-    @copydoc operator<<(basic_json&, std::istream&)
-    */
     friend std::istream& operator>>(std::istream& i, basic_json& j)
     {
-        j = parser(i).parse();
+        j = parser(input_adapter::create(i)).parse(false);
         return i;
     }
 
     /// @}
 
-    //////////////////////////////////////////
-    // binary serialization/deserialization //
-    //////////////////////////////////////////
-
-    /// @name binary serialization/deserialization support
-    /// @{
-
-  private:
-    /*!
-    @note Some code in the switch cases has been copied, because otherwise
-          copilers would complain about implicit fallthrough and there is no
-          portable attribute to mute such warnings.
-    */
-    template<typename T>
-    static void add_to_vector(std::vector<uint8_t>& vec, size_t bytes, const T number)
-    {
-        assert(bytes == 1 or bytes == 2 or bytes == 4 or bytes == 8);
-
-        switch (bytes)
-        {
-            case 8:
-            {
-                vec.push_back(static_cast<uint8_t>((static_cast<uint64_t>(number) >> 070) & 0xff));
-                vec.push_back(static_cast<uint8_t>((static_cast<uint64_t>(number) >> 060) & 0xff));
-                vec.push_back(static_cast<uint8_t>((static_cast<uint64_t>(number) >> 050) & 0xff));
-                vec.push_back(static_cast<uint8_t>((static_cast<uint64_t>(number) >> 040) & 0xff));
-                vec.push_back(static_cast<uint8_t>((number >> 030) & 0xff));
-                vec.push_back(static_cast<uint8_t>((number >> 020) & 0xff));
-                vec.push_back(static_cast<uint8_t>((number >> 010) & 0xff));
-                vec.push_back(static_cast<uint8_t>(number & 0xff));
-                break;
-            }
-
-            case 4:
-            {
-                vec.push_back(static_cast<uint8_t>((number >> 030) & 0xff));
-                vec.push_back(static_cast<uint8_t>((number >> 020) & 0xff));
-                vec.push_back(static_cast<uint8_t>((number >> 010) & 0xff));
-                vec.push_back(static_cast<uint8_t>(number & 0xff));
-                break;
-            }
-
-            case 2:
-            {
-                vec.push_back(static_cast<uint8_t>((number >> 010) & 0xff));
-                vec.push_back(static_cast<uint8_t>(number & 0xff));
-                break;
-            }
-
-            case 1:
-            {
-                vec.push_back(static_cast<uint8_t>(number & 0xff));
-                break;
-            }
-        }
-    }
-
-    /*!
-    @brief take sufficient bytes from a vector to fill an integer variable
-
-    In the context of binary serialization formats, we need to read several
-    bytes from a byte vector and combine them to multi-byte integral data
-    types.
-
-    @param[in] vec  byte vector to read from
-    @param[in] current_index  the position in the vector after which to read
-
-    @return the next sizeof(T) bytes from @a vec, in reverse order as T
-
-    @tparam T the integral return type
-
-    @throw std::out_of_range if there are less than sizeof(T)+1 bytes in the
-           vector @a vec to read
-
-    In the for loop, the bytes from the vector are copied in reverse order into
-    the return value. In the figures below, let sizeof(T)=4 and `i` be the loop
-    variable.
-
-    Precondition:
-
-    vec:   |   |   | a | b | c | d |      T: |   |   |   |   |
-                 ^               ^             ^                ^
-           current_index         i            ptr        sizeof(T)
-
-    Postcondition:
-
-    vec:   |   |   | a | b | c | d |      T: | d | c | b | a |
-                 ^   ^                                     ^
-                 |   i                                    ptr
-           current_index
-
-    @sa Code adapted from <http://stackoverflow.com/a/41031865/266378>.
-    */
-    template<typename T>
-    static T get_from_vector(const std::vector<uint8_t>& vec, const size_t current_index)
-    {
-        if (current_index + sizeof(T) + 1 > vec.size())
-        {
-            JSON_THROW(std::out_of_range("cannot read " + std::to_string(sizeof(T)) + " bytes from vector"));
-        }
-
-        T result;
-        auto* ptr = reinterpret_cast<uint8_t*>(&result);
-        for (size_t i = 0; i < sizeof(T); ++i)
-        {
-            *ptr++ = vec[current_index + sizeof(T) - i];
-        }
-        return result;
-    }
-
-    /*!
-    @brief create a MessagePack serialization of a given JSON value
-
-    This is a straightforward implementation of the MessagePack specification.
-
-    @param[in] j  JSON value to serialize
-    @param[in,out] v  byte vector to write the serialization to
-
-    @sa https://github.com/msgpack/msgpack/blob/master/spec.md
-    */
-    static void to_msgpack_internal(const basic_json& j, std::vector<uint8_t>& v)
-    {
-        switch (j.type())
-        {
-            case value_t::null:
-            {
-                // nil
-                v.push_back(0xc0);
-                break;
-            }
-
-            case value_t::boolean:
-            {
-                // true and false
-                v.push_back(j.m_value.boolean ? 0xc3 : 0xc2);
-                break;
-            }
-
-            case value_t::number_integer:
-            {
-                if (j.m_value.number_integer >= 0)
-                {
-                    // MessagePack does not differentiate between positive
-                    // signed integers and unsigned integers. Therefore, we
-                    // used the code from the value_t::number_unsigned case
-                    // here.
-                    if (j.m_value.number_unsigned < 128)
-                    {
-                        // positive fixnum
-                        add_to_vector(v, 1, j.m_value.number_unsigned);
-                    }
-                    else if (j.m_value.number_unsigned <= std::numeric_limits<uint8_t>::max())
-                    {
-                        // uint 8
-                        v.push_back(0xcc);
-                        add_to_vector(v, 1, j.m_value.number_unsigned);
-                    }
-                    else if (j.m_value.number_unsigned <= std::numeric_limits<uint16_t>::max())
-                    {
-                        // uint 16
-                        v.push_back(0xcd);
-                        add_to_vector(v, 2, j.m_value.number_unsigned);
-                    }
-                    else if (j.m_value.number_unsigned <= std::numeric_limits<uint32_t>::max())
-                    {
-                        // uint 32
-                        v.push_back(0xce);
-                        add_to_vector(v, 4, j.m_value.number_unsigned);
-                    }
-                    else if (j.m_value.number_unsigned <= std::numeric_limits<uint64_t>::max())
-                    {
-                        // uint 64
-                        v.push_back(0xcf);
-                        add_to_vector(v, 8, j.m_value.number_unsigned);
-                    }
-                }
-                else
-                {
-                    if (j.m_value.number_integer >= -32)
-                    {
-                        // negative fixnum
-                        add_to_vector(v, 1, j.m_value.number_integer);
-                    }
-                    else if (j.m_value.number_integer >= std::numeric_limits<int8_t>::min() and j.m_value.number_integer <= std::numeric_limits<int8_t>::max())
-                    {
-                        // int 8
-                        v.push_back(0xd0);
-                        add_to_vector(v, 1, j.m_value.number_integer);
-                    }
-                    else if (j.m_value.number_integer >= std::numeric_limits<int16_t>::min() and j.m_value.number_integer <= std::numeric_limits<int16_t>::max())
-                    {
-                        // int 16
-                        v.push_back(0xd1);
-                        add_to_vector(v, 2, j.m_value.number_integer);
-                    }
-                    else if (j.m_value.number_integer >= std::numeric_limits<int32_t>::min() and j.m_value.number_integer <= std::numeric_limits<int32_t>::max())
-                    {
-                        // int 32
-                        v.push_back(0xd2);
-                        add_to_vector(v, 4, j.m_value.number_integer);
-                    }
-                    else if (j.m_value.number_integer >= std::numeric_limits<int64_t>::min() and j.m_value.number_integer <= std::numeric_limits<int64_t>::max())
-                    {
-                        // int 64
-                        v.push_back(0xd3);
-                        add_to_vector(v, 8, j.m_value.number_integer);
-                    }
-                }
-                break;
-            }
-
-            case value_t::number_unsigned:
-            {
-                if (j.m_value.number_unsigned < 128)
-                {
-                    // positive fixnum
-                    add_to_vector(v, 1, j.m_value.number_unsigned);
-                }
-                else if (j.m_value.number_unsigned <= std::numeric_limits<uint8_t>::max())
-                {
-                    // uint 8
-                    v.push_back(0xcc);
-                    add_to_vector(v, 1, j.m_value.number_unsigned);
-                }
-                else if (j.m_value.number_unsigned <= std::numeric_limits<uint16_t>::max())
-                {
-                    // uint 16
-                    v.push_back(0xcd);
-                    add_to_vector(v, 2, j.m_value.number_unsigned);
-                }
-                else if (j.m_value.number_unsigned <= std::numeric_limits<uint32_t>::max())
-                {
-                    // uint 32
-                    v.push_back(0xce);
-                    add_to_vector(v, 4, j.m_value.number_unsigned);
-                }
-                else if (j.m_value.number_unsigned <= std::numeric_limits<uint64_t>::max())
-                {
-                    // uint 64
-                    v.push_back(0xcf);
-                    add_to_vector(v, 8, j.m_value.number_unsigned);
-                }
-                break;
-            }
-
-            case value_t::number_float:
-            {
-                // float 64
-                v.push_back(0xcb);
-                const auto* helper = reinterpret_cast<const uint8_t*>(&(j.m_value.number_float));
-                for (size_t i = 0; i < 8; ++i)
-                {
-                    v.push_back(helper[7 - i]);
-                }
-                break;
-            }
-
-            case value_t::string:
-            {
-                const auto N = j.m_value.string->size();
-                if (N <= 31)
-                {
-                    // fixstr
-                    v.push_back(static_cast<uint8_t>(0xa0 | N));
-                }
-                else if (N <= 255)
-                {
-                    // str 8
-                    v.push_back(0xd9);
-                    add_to_vector(v, 1, N);
-                }
-                else if (N <= 65535)
-                {
-                    // str 16
-                    v.push_back(0xda);
-                    add_to_vector(v, 2, N);
-                }
-                else if (N <= 4294967295)
-                {
-                    // str 32
-                    v.push_back(0xdb);
-                    add_to_vector(v, 4, N);
-                }
-
-                // append string
-                std::copy(j.m_value.string->begin(), j.m_value.string->end(),
-                          std::back_inserter(v));
-                break;
-            }
-
-            case value_t::array:
-            {
-                const auto N = j.m_value.array->size();
-                if (N <= 15)
-                {
-                    // fixarray
-                    v.push_back(static_cast<uint8_t>(0x90 | N));
-                }
-                else if (N <= 0xffff)
-                {
-                    // array 16
-                    v.push_back(0xdc);
-                    add_to_vector(v, 2, N);
-                }
-                else if (N <= 0xffffffff)
-                {
-                    // array 32
-                    v.push_back(0xdd);
-                    add_to_vector(v, 4, N);
-                }
-
-                // append each element
-                for (const auto& el : *j.m_value.array)
-                {
-                    to_msgpack_internal(el, v);
-                }
-                break;
-            }
-
-            case value_t::object:
-            {
-                const auto N = j.m_value.object->size();
-                if (N <= 15)
-                {
-                    // fixmap
-                    v.push_back(static_cast<uint8_t>(0x80 | (N & 0xf)));
-                }
-                else if (N <= 65535)
-                {
-                    // map 16
-                    v.push_back(0xde);
-                    add_to_vector(v, 2, N);
-                }
-                else if (N <= 4294967295)
-                {
-                    // map 32
-                    v.push_back(0xdf);
-                    add_to_vector(v, 4, N);
-                }
-
-                // append each element
-                for (const auto& el : *j.m_value.object)
-                {
-                    to_msgpack_internal(el.first, v);
-                    to_msgpack_internal(el.second, v);
-                }
-                break;
-            }
-
-            default:
-            {
-                break;
-            }
-        }
-    }
-
-    /*!
-    @brief create a CBOR serialization of a given JSON value
-
-    This is a straightforward implementation of the CBOR specification.
-
-    @param[in] j  JSON value to serialize
-    @param[in,out] v  byte vector to write the serialization to
-
-    @sa https://tools.ietf.org/html/rfc7049
-    */
-    static void to_cbor_internal(const basic_json& j, std::vector<uint8_t>& v)
-    {
-        switch (j.type())
-        {
-            case value_t::null:
-            {
-                v.push_back(0xf6);
-                break;
-            }
-
-            case value_t::boolean:
-            {
-                v.push_back(j.m_value.boolean ? 0xf5 : 0xf4);
-                break;
-            }
-
-            case value_t::number_integer:
-            {
-                if (j.m_value.number_integer >= 0)
-                {
-                    // CBOR does not differentiate between positive signed
-                    // integers and unsigned integers. Therefore, we used the
-                    // code from the value_t::number_unsigned case here.
-                    if (j.m_value.number_integer <= 0x17)
-                    {
-                        add_to_vector(v, 1, j.m_value.number_integer);
-                    }
-                    else if (j.m_value.number_integer <= std::numeric_limits<uint8_t>::max())
-                    {
-                        v.push_back(0x18);
-                        // one-byte uint8_t
-                        add_to_vector(v, 1, j.m_value.number_integer);
-                    }
-                    else if (j.m_value.number_integer <= std::numeric_limits<uint16_t>::max())
-                    {
-                        v.push_back(0x19);
-                        // two-byte uint16_t
-                        add_to_vector(v, 2, j.m_value.number_integer);
-                    }
-                    else if (j.m_value.number_integer <= std::numeric_limits<uint32_t>::max())
-                    {
-                        v.push_back(0x1a);
-                        // four-byte uint32_t
-                        add_to_vector(v, 4, j.m_value.number_integer);
-                    }
-                    else
-                    {
-                        v.push_back(0x1b);
-                        // eight-byte uint64_t
-                        add_to_vector(v, 8, j.m_value.number_integer);
-                    }
-                }
-                else
-                {
-                    // The conversions below encode the sign in the first
-                    // byte, and the value is converted to a positive number.
-                    const auto positive_number = -1 - j.m_value.number_integer;
-                    if (j.m_value.number_integer >= -24)
-                    {
-                        v.push_back(static_cast<uint8_t>(0x20 + positive_number));
-                    }
-                    else if (positive_number <= std::numeric_limits<uint8_t>::max())
-                    {
-                        // int 8
-                        v.push_back(0x38);
-                        add_to_vector(v, 1, positive_number);
-                    }
-                    else if (positive_number <= std::numeric_limits<uint16_t>::max())
-                    {
-                        // int 16
-                        v.push_back(0x39);
-                        add_to_vector(v, 2, positive_number);
-                    }
-                    else if (positive_number <= std::numeric_limits<uint32_t>::max())
-                    {
-                        // int 32
-                        v.push_back(0x3a);
-                        add_to_vector(v, 4, positive_number);
-                    }
-                    else
-                    {
-                        // int 64
-                        v.push_back(0x3b);
-                        add_to_vector(v, 8, positive_number);
-                    }
-                }
-                break;
-            }
-
-            case value_t::number_unsigned:
-            {
-                if (j.m_value.number_unsigned <= 0x17)
-                {
-                    v.push_back(static_cast<uint8_t>(j.m_value.number_unsigned));
-                }
-                else if (j.m_value.number_unsigned <= 0xff)
-                {
-                    v.push_back(0x18);
-                    // one-byte uint8_t
-                    add_to_vector(v, 1, j.m_value.number_unsigned);
-                }
-                else if (j.m_value.number_unsigned <= 0xffff)
-                {
-                    v.push_back(0x19);
-                    // two-byte uint16_t
-                    add_to_vector(v, 2, j.m_value.number_unsigned);
-                }
-                else if (j.m_value.number_unsigned <= 0xffffffff)
-                {
-                    v.push_back(0x1a);
-                    // four-byte uint32_t
-                    add_to_vector(v, 4, j.m_value.number_unsigned);
-                }
-                else if (j.m_value.number_unsigned <= 0xffffffffffffffff)
-                {
-                    v.push_back(0x1b);
-                    // eight-byte uint64_t
-                    add_to_vector(v, 8, j.m_value.number_unsigned);
-                }
-                break;
-            }
-
-            case value_t::number_float:
-            {
-                // Double-Precision Float
-                v.push_back(0xfb);
-                const auto* helper = reinterpret_cast<const uint8_t*>(&(j.m_value.number_float));
-                for (size_t i = 0; i < 8; ++i)
-                {
-                    v.push_back(helper[7 - i]);
-                }
-                break;
-            }
-
-            case value_t::string:
-            {
-                const auto N = j.m_value.string->size();
-                if (N <= 0x17)
-                {
-                    v.push_back(0x60 + static_cast<uint8_t>(N));  // 1 byte for string + size
-                }
-                else if (N <= 0xff)
-                {
-                    v.push_back(0x78);  // one-byte uint8_t for N
-                    add_to_vector(v, 1, N);
-                }
-                else if (N <= 0xffff)
-                {
-                    v.push_back(0x79);  // two-byte uint16_t for N
-                    add_to_vector(v, 2, N);
-                }
-                else if (N <= 0xffffffff)
-                {
-                    v.push_back(0x7a); // four-byte uint32_t for N
-                    add_to_vector(v, 4, N);
-                }
-                // LCOV_EXCL_START
-                else if (N <= 0xffffffffffffffff)
-                {
-                    v.push_back(0x7b);  // eight-byte uint64_t for N
-                    add_to_vector(v, 8, N);
-                }
-                // LCOV_EXCL_STOP
-
-                // append string
-                std::copy(j.m_value.string->begin(), j.m_value.string->end(),
-                          std::back_inserter(v));
-                break;
-            }
-
-            case value_t::array:
-            {
-                const auto N = j.m_value.array->size();
-                if (N <= 0x17)
-                {
-                    v.push_back(0x80 + static_cast<uint8_t>(N));  // 1 byte for array + size
-                }
-                else if (N <= 0xff)
-                {
-                    v.push_back(0x98);  // one-byte uint8_t for N
-                    add_to_vector(v, 1, N);
-                }
-                else if (N <= 0xffff)
-                {
-                    v.push_back(0x99);  // two-byte uint16_t for N
-                    add_to_vector(v, 2, N);
-                }
-                else if (N <= 0xffffffff)
-                {
-                    v.push_back(0x9a);  // four-byte uint32_t for N
-                    add_to_vector(v, 4, N);
-                }
-                // LCOV_EXCL_START
-                else if (N <= 0xffffffffffffffff)
-                {
-                    v.push_back(0x9b);  // eight-byte uint64_t for N
-                    add_to_vector(v, 8, N);
-                }
-                // LCOV_EXCL_STOP
-
-                // append each element
-                for (const auto& el : *j.m_value.array)
-                {
-                    to_cbor_internal(el, v);
-                }
-                break;
-            }
-
-            case value_t::object:
-            {
-                const auto N = j.m_value.object->size();
-                if (N <= 0x17)
-                {
-                    v.push_back(0xa0 + static_cast<uint8_t>(N));  // 1 byte for object + size
-                }
-                else if (N <= 0xff)
-                {
-                    v.push_back(0xb8);
-                    add_to_vector(v, 1, N);  // one-byte uint8_t for N
-                }
-                else if (N <= 0xffff)
-                {
-                    v.push_back(0xb9);
-                    add_to_vector(v, 2, N);  // two-byte uint16_t for N
-                }
-                else if (N <= 0xffffffff)
-                {
-                    v.push_back(0xba);
-                    add_to_vector(v, 4, N);  // four-byte uint32_t for N
-                }
-                // LCOV_EXCL_START
-                else if (N <= 0xffffffffffffffff)
-                {
-                    v.push_back(0xbb);
-                    add_to_vector(v, 8, N);  // eight-byte uint64_t for N
-                }
-                // LCOV_EXCL_STOP
-
-                // append each element
-                for (const auto& el : *j.m_value.object)
-                {
-                    to_cbor_internal(el.first, v);
-                    to_cbor_internal(el.second, v);
-                }
-                break;
-            }
-
-            default:
-            {
-                break;
-            }
-        }
-    }
-
-
-    /*
-    @brief checks if given lengths do not exceed the size of a given vector
-
-    To secure the access to the byte vector during CBOR/MessagePack
-    deserialization, bytes are copied from the vector into buffers. This
-    function checks if the number of bytes to copy (@a len) does not exceed
-    the size @s size of the vector. Additionally, an @a offset is given from
-    where to start reading the bytes.
-
-    This function checks whether reading the bytes is safe; that is, offset is
-    a valid index in the vector, offset+len
-
-    @param[in] size    size of the byte vector
-    @param[in] len     number of bytes to read
-    @param[in] offset  offset where to start reading
-
-    vec:  x x x x x X X X X X
-          ^         ^         ^
-          0         offset    len
-
-    @throws out_of_range if `len > v.size()`
-    */
-    static void check_length(const size_t size, const size_t len, const size_t offset)
-    {
-        // simple case: requested length is greater than the vector's length
-        if (len > size or offset > size)
-        {
-            JSON_THROW(std::out_of_range("len out of range"));
-        }
-
-        // second case: adding offset would result in overflow
-        if ((size > (std::numeric_limits<size_t>::max() - offset)))
-        {
-            JSON_THROW(std::out_of_range("len+offset out of range"));
-        }
-
-        // last case: reading past the end of the vector
-        if (len + offset > size)
-        {
-            JSON_THROW(std::out_of_range("len+offset out of range"));
-        }
-    }
-
-    /*!
-    @brief create a JSON value from a given MessagePack vector
-
-    @param[in] v  MessagePack serialization
-    @param[in] idx  byte index to start reading from @a v
-
-    @return deserialized JSON value
-
-    @throw std::invalid_argument if unsupported features from MessagePack were
-    used in the given vector @a v or if the input is not valid MessagePack
-    @throw std::out_of_range if the given vector ends prematurely
-
-    @sa https://github.com/msgpack/msgpack/blob/master/spec.md
-    */
-    static basic_json from_msgpack_internal(const std::vector<uint8_t>& v, size_t& idx)
-    {
-        // make sure reading 1 byte is safe
-        check_length(v.size(), 1, idx);
-
-        // store and increment index
-        const size_t current_idx = idx++;
-
-        if (v[current_idx] <= 0xbf)
-        {
-            if (v[current_idx] <= 0x7f) // positive fixint
-            {
-                return v[current_idx];
-            }
-            if (v[current_idx] <= 0x8f) // fixmap
-            {
-                basic_json result = value_t::object;
-                const size_t len = v[current_idx] & 0x0f;
-                for (size_t i = 0; i < len; ++i)
-                {
-                    std::string key = from_msgpack_internal(v, idx);
-                    result[key] = from_msgpack_internal(v, idx);
-                }
-                return result;
-            }
-            else if (v[current_idx] <= 0x9f) // fixarray
-            {
-                basic_json result = value_t::array;
-                const size_t len = v[current_idx] & 0x0f;
-                for (size_t i = 0; i < len; ++i)
-                {
-                    result.push_back(from_msgpack_internal(v, idx));
-                }
-                return result;
-            }
-            else // fixstr
-            {
-                const size_t len = v[current_idx] & 0x1f;
-                const size_t offset = current_idx + 1;
-                idx += len; // skip content bytes
-                check_length(v.size(), len, offset);
-                return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
-            }
-        }
-        else if (v[current_idx] >= 0xe0) // negative fixint
-        {
-            return static_cast<int8_t>(v[current_idx]);
-        }
-        else
-        {
-            switch (v[current_idx])
-            {
-                case 0xc0: // nil
-                {
-                    return value_t::null;
-                }
-
-                case 0xc2: // false
-                {
-                    return false;
-                }
-
-                case 0xc3: // true
-                {
-                    return true;
-                }
-
-                case 0xca: // float 32
-                {
-                    // copy bytes in reverse order into the double variable
-                    float res;
-                    for (size_t byte = 0; byte < sizeof(float); ++byte)
-                    {
-                        reinterpret_cast<uint8_t*>(&res)[sizeof(float) - byte - 1] = v.at(current_idx + 1 + byte);
-                    }
-                    idx += sizeof(float); // skip content bytes
-                    return res;
-                }
-
-                case 0xcb: // float 64
-                {
-                    // copy bytes in reverse order into the double variable
-                    double res;
-                    for (size_t byte = 0; byte < sizeof(double); ++byte)
-                    {
-                        reinterpret_cast<uint8_t*>(&res)[sizeof(double) - byte - 1] = v.at(current_idx + 1 + byte);
-                    }
-                    idx += sizeof(double); // skip content bytes
-                    return res;
-                }
-
-                case 0xcc: // uint 8
-                {
-                    idx += 1; // skip content byte
-                    return get_from_vector<uint8_t>(v, current_idx);
-                }
-
-                case 0xcd: // uint 16
-                {
-                    idx += 2; // skip 2 content bytes
-                    return get_from_vector<uint16_t>(v, current_idx);
-                }
-
-                case 0xce: // uint 32
-                {
-                    idx += 4; // skip 4 content bytes
-                    return get_from_vector<uint32_t>(v, current_idx);
-                }
-
-                case 0xcf: // uint 64
-                {
-                    idx += 8; // skip 8 content bytes
-                    return get_from_vector<uint64_t>(v, current_idx);
-                }
-
-                case 0xd0: // int 8
-                {
-                    idx += 1; // skip content byte
-                    return get_from_vector<int8_t>(v, current_idx);
-                }
-
-                case 0xd1: // int 16
-                {
-                    idx += 2; // skip 2 content bytes
-                    return get_from_vector<int16_t>(v, current_idx);
-                }
-
-                case 0xd2: // int 32
-                {
-                    idx += 4; // skip 4 content bytes
-                    return get_from_vector<int32_t>(v, current_idx);
-                }
-
-                case 0xd3: // int 64
-                {
-                    idx += 8; // skip 8 content bytes
-                    return get_from_vector<int64_t>(v, current_idx);
-                }
-
-                case 0xd9: // str 8
-                {
-                    const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
-                    const size_t offset = current_idx + 2;
-                    idx += len + 1; // skip size byte + content bytes
-                    check_length(v.size(), len, offset);
-                    return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
-                }
-
-                case 0xda: // str 16
-                {
-                    const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
-                    const size_t offset = current_idx + 3;
-                    idx += len + 2; // skip 2 size bytes + content bytes
-                    check_length(v.size(), len, offset);
-                    return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
-                }
-
-                case 0xdb: // str 32
-                {
-                    const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
-                    const size_t offset = current_idx + 5;
-                    idx += len + 4; // skip 4 size bytes + content bytes
-                    check_length(v.size(), len, offset);
-                    return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
-                }
-
-                case 0xdc: // array 16
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
-                    idx += 2; // skip 2 size bytes
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(from_msgpack_internal(v, idx));
-                    }
-                    return result;
-                }
-
-                case 0xdd: // array 32
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
-                    idx += 4; // skip 4 size bytes
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(from_msgpack_internal(v, idx));
-                    }
-                    return result;
-                }
-
-                case 0xde: // map 16
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
-                    idx += 2; // skip 2 size bytes
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        std::string key = from_msgpack_internal(v, idx);
-                        result[key] = from_msgpack_internal(v, idx);
-                    }
-                    return result;
-                }
-
-                case 0xdf: // map 32
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
-                    idx += 4; // skip 4 size bytes
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        std::string key = from_msgpack_internal(v, idx);
-                        result[key] = from_msgpack_internal(v, idx);
-                    }
-                    return result;
-                }
-
-                default:
-                {
-                    JSON_THROW(std::invalid_argument("error parsing a msgpack @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast<int>(v[current_idx]))));
-                }
-            }
-        }
-    }
-
-    /*!
-    @brief create a JSON value from a given CBOR vector
-
-    @param[in] v  CBOR serialization
-    @param[in] idx  byte index to start reading from @a v
-
-    @return deserialized JSON value
-
-    @throw std::invalid_argument if unsupported features from CBOR were used in
-    the given vector @a v or if the input is not valid CBOR
-    @throw std::out_of_range if the given vector ends prematurely
-
-    @sa https://tools.ietf.org/html/rfc7049
-    */
-    static basic_json from_cbor_internal(const std::vector<uint8_t>& v, size_t& idx)
-    {
-        // store and increment index
-        const size_t current_idx = idx++;
-
-        switch (v.at(current_idx))
-        {
-            // Integer 0x00..0x17 (0..23)
-            case 0x00:
-            case 0x01:
-            case 0x02:
-            case 0x03:
-            case 0x04:
-            case 0x05:
-            case 0x06:
-            case 0x07:
-            case 0x08:
-            case 0x09:
-            case 0x0a:
-            case 0x0b:
-            case 0x0c:
-            case 0x0d:
-            case 0x0e:
-            case 0x0f:
-            case 0x10:
-            case 0x11:
-            case 0x12:
-            case 0x13:
-            case 0x14:
-            case 0x15:
-            case 0x16:
-            case 0x17:
-            {
-                return v[current_idx];
-            }
-
-            case 0x18: // Unsigned integer (one-byte uint8_t follows)
-            {
-                idx += 1; // skip content byte
-                return get_from_vector<uint8_t>(v, current_idx);
-            }
-
-            case 0x19: // Unsigned integer (two-byte uint16_t follows)
-            {
-                idx += 2; // skip 2 content bytes
-                return get_from_vector<uint16_t>(v, current_idx);
-            }
-
-            case 0x1a: // Unsigned integer (four-byte uint32_t follows)
-            {
-                idx += 4; // skip 4 content bytes
-                return get_from_vector<uint32_t>(v, current_idx);
-            }
-
-            case 0x1b: // Unsigned integer (eight-byte uint64_t follows)
-            {
-                idx += 8; // skip 8 content bytes
-                return get_from_vector<uint64_t>(v, current_idx);
-            }
-
-            // Negative integer -1-0x00..-1-0x17 (-1..-24)
-            case 0x20:
-            case 0x21:
-            case 0x22:
-            case 0x23:
-            case 0x24:
-            case 0x25:
-            case 0x26:
-            case 0x27:
-            case 0x28:
-            case 0x29:
-            case 0x2a:
-            case 0x2b:
-            case 0x2c:
-            case 0x2d:
-            case 0x2e:
-            case 0x2f:
-            case 0x30:
-            case 0x31:
-            case 0x32:
-            case 0x33:
-            case 0x34:
-            case 0x35:
-            case 0x36:
-            case 0x37:
-            {
-                return static_cast<int8_t>(0x20 - 1 - v[current_idx]);
-            }
-
-            case 0x38: // Negative integer (one-byte uint8_t follows)
-            {
-                idx += 1; // skip content byte
-                // must be uint8_t !
-                return static_cast<number_integer_t>(-1) - get_from_vector<uint8_t>(v, current_idx);
-            }
-
-            case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
-            {
-                idx += 2; // skip 2 content bytes
-                return static_cast<number_integer_t>(-1) - get_from_vector<uint16_t>(v, current_idx);
-            }
-
-            case 0x3a: // Negative integer -1-n (four-byte uint32_t follows)
-            {
-                idx += 4; // skip 4 content bytes
-                return static_cast<number_integer_t>(-1) - get_from_vector<uint32_t>(v, current_idx);
-            }
-
-            case 0x3b: // Negative integer -1-n (eight-byte uint64_t follows)
-            {
-                idx += 8; // skip 8 content bytes
-                return static_cast<number_integer_t>(-1) - static_cast<number_integer_t>(get_from_vector<uint64_t>(v, current_idx));
-            }
-
-            // UTF-8 string (0x00..0x17 bytes follow)
-            case 0x60:
-            case 0x61:
-            case 0x62:
-            case 0x63:
-            case 0x64:
-            case 0x65:
-            case 0x66:
-            case 0x67:
-            case 0x68:
-            case 0x69:
-            case 0x6a:
-            case 0x6b:
-            case 0x6c:
-            case 0x6d:
-            case 0x6e:
-            case 0x6f:
-            case 0x70:
-            case 0x71:
-            case 0x72:
-            case 0x73:
-            case 0x74:
-            case 0x75:
-            case 0x76:
-            case 0x77:
-            {
-                const auto len = static_cast<size_t>(v[current_idx] - 0x60);
-                const size_t offset = current_idx + 1;
-                idx += len; // skip content bytes
-                check_length(v.size(), len, offset);
-                return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
-            }
-
-            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
-            {
-                const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
-                const size_t offset = current_idx + 2;
-                idx += len + 1; // skip size byte + content bytes
-                check_length(v.size(), len, offset);
-                return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
-            }
-
-            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
-            {
-                const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
-                const size_t offset = current_idx + 3;
-                idx += len + 2; // skip 2 size bytes + content bytes
-                check_length(v.size(), len, offset);
-                return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
-            }
-
-            case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
-            {
-                const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
-                const size_t offset = current_idx + 5;
-                idx += len + 4; // skip 4 size bytes + content bytes
-                check_length(v.size(), len, offset);
-                return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
-            }
-
-            case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
-            {
-                const auto len = static_cast<size_t>(get_from_vector<uint64_t>(v, current_idx));
-                const size_t offset = current_idx + 9;
-                idx += len + 8; // skip 8 size bytes + content bytes
-                check_length(v.size(), len, offset);
-                return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
-            }
-
-            case 0x7f: // UTF-8 string (indefinite length)
-            {
-                std::string result;
-                while (v.at(idx) != 0xff)
-                {
-                    string_t s = from_cbor_internal(v, idx);
-                    result += s;
-                }
-                // skip break byte (0xFF)
-                idx += 1;
-                return result;
-            }
-
-            // array (0x00..0x17 data items follow)
-            case 0x80:
-            case 0x81:
-            case 0x82:
-            case 0x83:
-            case 0x84:
-            case 0x85:
-            case 0x86:
-            case 0x87:
-            case 0x88:
-            case 0x89:
-            case 0x8a:
-            case 0x8b:
-            case 0x8c:
-            case 0x8d:
-            case 0x8e:
-            case 0x8f:
-            case 0x90:
-            case 0x91:
-            case 0x92:
-            case 0x93:
-            case 0x94:
-            case 0x95:
-            case 0x96:
-            case 0x97:
-            {
-                basic_json result = value_t::array;
-                const auto len = static_cast<size_t>(v[current_idx] - 0x80);
-                for (size_t i = 0; i < len; ++i)
-                {
-                    result.push_back(from_cbor_internal(v, idx));
-                }
-                return result;
-            }
-
-            case 0x98: // array (one-byte uint8_t for n follows)
-            {
-                basic_json result = value_t::array;
-                const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
-                idx += 1; // skip 1 size byte
-                for (size_t i = 0; i < len; ++i)
-                {
-                    result.push_back(from_cbor_internal(v, idx));
-                }
-                return result;
-            }
-
-            case 0x99: // array (two-byte uint16_t for n follow)
-            {
-                basic_json result = value_t::array;
-                const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
-                idx += 2; // skip 4 size bytes
-                for (size_t i = 0; i < len; ++i)
-                {
-                    result.push_back(from_cbor_internal(v, idx));
-                }
-                return result;
-            }
-
-            case 0x9a: // array (four-byte uint32_t for n follow)
-            {
-                basic_json result = value_t::array;
-                const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
-                idx += 4; // skip 4 size bytes
-                for (size_t i = 0; i < len; ++i)
-                {
-                    result.push_back(from_cbor_internal(v, idx));
-                }
-                return result;
-            }
-
-            case 0x9b: // array (eight-byte uint64_t for n follow)
-            {
-                basic_json result = value_t::array;
-                const auto len = static_cast<size_t>(get_from_vector<uint64_t>(v, current_idx));
-                idx += 8; // skip 8 size bytes
-                for (size_t i = 0; i < len; ++i)
-                {
-                    result.push_back(from_cbor_internal(v, idx));
-                }
-                return result;
-            }
-
-            case 0x9f: // array (indefinite length)
-            {
-                basic_json result = value_t::array;
-                while (v.at(idx) != 0xff)
-                {
-                    result.push_back(from_cbor_internal(v, idx));
-                }
-                // skip break byte (0xFF)
-                idx += 1;
-                return result;
-            }
-
-            // map (0x00..0x17 pairs of data items follow)
-            case 0xa0:
-            case 0xa1:
-            case 0xa2:
-            case 0xa3:
-            case 0xa4:
-            case 0xa5:
-            case 0xa6:
-            case 0xa7:
-            case 0xa8:
-            case 0xa9:
-            case 0xaa:
-            case 0xab:
-            case 0xac:
-            case 0xad:
-            case 0xae:
-            case 0xaf:
-            case 0xb0:
-            case 0xb1:
-            case 0xb2:
-            case 0xb3:
-            case 0xb4:
-            case 0xb5:
-            case 0xb6:
-            case 0xb7:
-            {
-                basic_json result = value_t::object;
-                const auto len = static_cast<size_t>(v[current_idx] - 0xa0);
-                for (size_t i = 0; i < len; ++i)
-                {
-                    std::string key = from_cbor_internal(v, idx);
-                    result[key] = from_cbor_internal(v, idx);
-                }
-                return result;
-            }
-
-            case 0xb8: // map (one-byte uint8_t for n follows)
-            {
-                basic_json result = value_t::object;
-                const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
-                idx += 1; // skip 1 size byte
-                for (size_t i = 0; i < len; ++i)
-                {
-                    std::string key = from_cbor_internal(v, idx);
-                    result[key] = from_cbor_internal(v, idx);
-                }
-                return result;
-            }
-
-            case 0xb9: // map (two-byte uint16_t for n follow)
-            {
-                basic_json result = value_t::object;
-                const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
-                idx += 2; // skip 2 size bytes
-                for (size_t i = 0; i < len; ++i)
-                {
-                    std::string key = from_cbor_internal(v, idx);
-                    result[key] = from_cbor_internal(v, idx);
-                }
-                return result;
-            }
-
-            case 0xba: // map (four-byte uint32_t for n follow)
-            {
-                basic_json result = value_t::object;
-                const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
-                idx += 4; // skip 4 size bytes
-                for (size_t i = 0; i < len; ++i)
-                {
-                    std::string key = from_cbor_internal(v, idx);
-                    result[key] = from_cbor_internal(v, idx);
-                }
-                return result;
-            }
-
-            case 0xbb: // map (eight-byte uint64_t for n follow)
-            {
-                basic_json result = value_t::object;
-                const auto len = static_cast<size_t>(get_from_vector<uint64_t>(v, current_idx));
-                idx += 8; // skip 8 size bytes
-                for (size_t i = 0; i < len; ++i)
-                {
-                    std::string key = from_cbor_internal(v, idx);
-                    result[key] = from_cbor_internal(v, idx);
-                }
-                return result;
-            }
-
-            case 0xbf: // map (indefinite length)
-            {
-                basic_json result = value_t::object;
-                while (v.at(idx) != 0xff)
-                {
-                    std::string key = from_cbor_internal(v, idx);
-                    result[key] = from_cbor_internal(v, idx);
-                }
-                // skip break byte (0xFF)
-                idx += 1;
-                return result;
-            }
-
-            case 0xf4: // false
-            {
-                return false;
-            }
-
-            case 0xf5: // true
-            {
-                return true;
-            }
-
-            case 0xf6: // null
-            {
-                return value_t::null;
-            }
-
-            case 0xf9: // Half-Precision Float (two-byte IEEE 754)
-            {
-                idx += 2; // skip two content bytes
-
-                // code from RFC 7049, Appendix D, Figure 3:
-                // As half-precision floating-point numbers were only added to
-                // IEEE 754 in 2008, today's programming platforms often still
-                // only have limited support for them. It is very easy to
-                // include at least decoding support for them even without such
-                // support. An example of a small decoder for half-precision
-                // floating-point numbers in the C language is shown in Fig. 3.
-                const int half = (v.at(current_idx + 1) << 8) + v.at(current_idx + 2);
-                const int exp = (half >> 10) & 0x1f;
-                const int mant = half & 0x3ff;
-                double val;
-                if (exp == 0)
-                {
-                    val = std::ldexp(mant, -24);
-                }
-                else if (exp != 31)
-                {
-                    val = std::ldexp(mant + 1024, exp - 25);
-                }
-                else
-                {
-                    val = mant == 0
-                          ? std::numeric_limits<double>::infinity()
-                          : std::numeric_limits<double>::quiet_NaN();
-                }
-                return (half & 0x8000) != 0 ? -val : val;
-            }
-
-            case 0xfa: // Single-Precision Float (four-byte IEEE 754)
-            {
-                // copy bytes in reverse order into the float variable
-                float res;
-                for (size_t byte = 0; byte < sizeof(float); ++byte)
-                {
-                    reinterpret_cast<uint8_t*>(&res)[sizeof(float) - byte - 1] = v.at(current_idx + 1 + byte);
-                }
-                idx += sizeof(float); // skip content bytes
-                return res;
-            }
-
-            case 0xfb: // Double-Precision Float (eight-byte IEEE 754)
-            {
-                // copy bytes in reverse order into the double variable
-                double res;
-                for (size_t byte = 0; byte < sizeof(double); ++byte)
-                {
-                    reinterpret_cast<uint8_t*>(&res)[sizeof(double) - byte - 1] = v.at(current_idx + 1 + byte);
-                }
-                idx += sizeof(double); // skip content bytes
-                return res;
-            }
-
-            default: // anything else (0xFF is handled inside the other types)
-            {
-                JSON_THROW(std::invalid_argument("error parsing a CBOR @ " + std::to_string(current_idx) + ": " + std::to_string(static_cast<int>(v[current_idx]))));
-            }
-        }
-    }
-
-  public:
-    /*!
-    @brief create a MessagePack serialization of a given JSON value
-
-    Serializes a given JSON value @a j to a byte vector using the MessagePack
-    serialization format. MessagePack is a binary serialization format which
-    aims to be more compact than JSON itself, yet more efficient to parse.
-
-    @param[in] j  JSON value to serialize
-    @return MessagePack serialization as byte vector
-
-    @complexity Linear in the size of the JSON value @a j.
-
-    @liveexample{The example shows the serialization of a JSON value to a byte
-    vector in MessagePack format.,to_msgpack}
-
-    @sa http://msgpack.org
-    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
-        analogous deserialization
-    @sa @ref to_cbor(const basic_json& for the related CBOR format
-
-    @since version 2.0.9
-    */
-    static std::vector<uint8_t> to_msgpack(const basic_json& j)
-    {
-        std::vector<uint8_t> result;
-        to_msgpack_internal(j, result);
-        return result;
-    }
-
-    /*!
-    @brief create a JSON value from a byte vector in MessagePack format
-
-    Deserializes a given byte vector @a v to a JSON value using the MessagePack
-    serialization format.
-
-    @param[in] v  a byte vector in MessagePack format
-    @param[in] start_index the index to start reading from @a v (0 by default)
-    @return deserialized JSON value
-
-    @throw std::invalid_argument if unsupported features from MessagePack were
-    used in the given vector @a v or if the input is not valid MessagePack
-    @throw std::out_of_range if the given vector ends prematurely
-
-    @complexity Linear in the size of the byte vector @a v.
-
-    @liveexample{The example shows the deserialization of a byte vector in
-    MessagePack format to a JSON value.,from_msgpack}
-
-    @sa http://msgpack.org
-    @sa @ref to_msgpack(const basic_json&) for the analogous serialization
-    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
-        related CBOR format
-
-    @since version 2.0.9, parameter @a start_index since 2.1.1
-    */
-    static basic_json from_msgpack(const std::vector<uint8_t>& v,
-                                   const size_t start_index = 0)
-    {
-        size_t i = start_index;
-        return from_msgpack_internal(v, i);
-    }
-
-    /*!
-    @brief create a MessagePack serialization of a given JSON value
-
-    Serializes a given JSON value @a j to a byte vector using the CBOR (Concise
-    Binary Object Representation) serialization format. CBOR is a binary
-    serialization format which aims to be more compact than JSON itself, yet
-    more efficient to parse.
-
-    @param[in] j  JSON value to serialize
-    @return MessagePack serialization as byte vector
-
-    @complexity Linear in the size of the JSON value @a j.
-
-    @liveexample{The example shows the serialization of a JSON value to a byte
-    vector in CBOR format.,to_cbor}
-
-    @sa http://cbor.io
-    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
-        analogous deserialization
-    @sa @ref to_msgpack(const basic_json& for the related MessagePack format
-
-    @since version 2.0.9
-    */
-    static std::vector<uint8_t> to_cbor(const basic_json& j)
-    {
-        std::vector<uint8_t> result;
-        to_cbor_internal(j, result);
-        return result;
-    }
-
-    /*!
-    @brief create a JSON value from a byte vector in CBOR format
-
-    Deserializes a given byte vector @a v to a JSON value using the CBOR
-    (Concise Binary Object Representation) serialization format.
-
-    @param[in] v  a byte vector in CBOR format
-    @param[in] start_index the index to start reading from @a v (0 by default)
-    @return deserialized JSON value
-
-    @throw std::invalid_argument if unsupported features from CBOR were used in
-    the given vector @a v or if the input is not valid MessagePack
-    @throw std::out_of_range if the given vector ends prematurely
-
-    @complexity Linear in the size of the byte vector @a v.
-
-    @liveexample{The example shows the deserialization of a byte vector in CBOR
-    format to a JSON value.,from_cbor}
-
-    @sa http://cbor.io
-    @sa @ref to_cbor(const basic_json&) for the analogous serialization
-    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
-        related MessagePack format
-
-    @since version 2.0.9, parameter @a start_index since 2.1.1
-    */
-    static basic_json from_cbor(const std::vector<uint8_t>& v,
-                                const size_t start_index = 0)
-    {
-        size_t i = start_index;
-        return from_cbor_internal(v, i);
-    }
-
-    /// @}
-
     ///////////////////////////
     // convenience functions //
     ///////////////////////////
@@ -8085,464 +7833,6 @@ class basic_json
         }
     }
 
-  private:
-    /*!
-    @brief calculates the extra space to escape a JSON string
-
-    @param[in] s  the string to escape
-    @return the number of characters required to escape string @a s
-
-    @complexity Linear in the length of string @a s.
-    */
-    static std::size_t extra_space(const string_t& s) noexcept
-    {
-        return std::accumulate(s.begin(), s.end(), size_t{},
-                               [](size_t res, typename string_t::value_type c)
-        {
-            switch (c)
-            {
-                case '"':
-                case '\\':
-                case '\b':
-                case '\f':
-                case '\n':
-                case '\r':
-                case '\t':
-                {
-                    // from c (1 byte) to \x (2 bytes)
-                    return res + 1;
-                }
-
-                default:
-                {
-                    if (c >= 0x00 and c <= 0x1f)
-                    {
-                        // from c (1 byte) to \uxxxx (6 bytes)
-                        return res + 5;
-                    }
-
-                    return res;
-                }
-            }
-        });
-    }
-
-    /*!
-    @brief escape a string
-
-    Escape a string by replacing certain special characters by a sequence of
-    an escape character (backslash) and another character and other control
-    characters by a sequence of "\u" followed by a four-digit hex
-    representation.
-
-    @param[in] s  the string to escape
-    @return  the escaped string
-
-    @complexity Linear in the length of string @a s.
-    */
-    static string_t escape_string(const string_t& s)
-    {
-        const auto space = extra_space(s);
-        if (space == 0)
-        {
-            return s;
-        }
-
-        // create a result string of necessary size
-        string_t result(s.size() + space, '\\');
-        std::size_t pos = 0;
-
-        for (const auto& c : s)
-        {
-            switch (c)
-            {
-                // quotation mark (0x22)
-                case '"':
-                {
-                    result[pos + 1] = '"';
-                    pos += 2;
-                    break;
-                }
-
-                // reverse solidus (0x5c)
-                case '\\':
-                {
-                    // nothing to change
-                    pos += 2;
-                    break;
-                }
-
-                // backspace (0x08)
-                case '\b':
-                {
-                    result[pos + 1] = 'b';
-                    pos += 2;
-                    break;
-                }
-
-                // formfeed (0x0c)
-                case '\f':
-                {
-                    result[pos + 1] = 'f';
-                    pos += 2;
-                    break;
-                }
-
-                // newline (0x0a)
-                case '\n':
-                {
-                    result[pos + 1] = 'n';
-                    pos += 2;
-                    break;
-                }
-
-                // carriage return (0x0d)
-                case '\r':
-                {
-                    result[pos + 1] = 'r';
-                    pos += 2;
-                    break;
-                }
-
-                // horizontal tab (0x09)
-                case '\t':
-                {
-                    result[pos + 1] = 't';
-                    pos += 2;
-                    break;
-                }
-
-                default:
-                {
-                    if (c >= 0x00 and c <= 0x1f)
-                    {
-                        // convert a number 0..15 to its hex representation
-                        // (0..f)
-                        static const char hexify[16] =
-                        {
-                            '0', '1', '2', '3', '4', '5', '6', '7',
-                            '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
-                        };
-
-                        // print character c as \uxxxx
-                        for (const char m :
-                    { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f]
-                        })
-                        {
-                            result[++pos] = m;
-                        }
-
-                        ++pos;
-                    }
-                    else
-                    {
-                        // all other characters are added as-is
-                        result[pos++] = c;
-                    }
-                    break;
-                }
-            }
-        }
-
-        return result;
-    }
-
-
-    /*!
-    @brief locale-independent serialization for built-in arithmetic types
-    */
-    struct numtostr
-    {
-      public:
-        template<typename NumberType>
-        numtostr(NumberType value)
-        {
-            x_write(value, std::is_integral<NumberType>());
-        }
-
-        const char* c_str() const
-        {
-            return m_buf.data();
-        }
-
-      private:
-        /// a (hopefully) large enough character buffer
-        std::array < char, 64 > m_buf{{}};
-
-        template<typename NumberType>
-        void x_write(NumberType x, /*is_integral=*/std::true_type)
-        {
-            // special case for "0"
-            if (x == 0)
-            {
-                m_buf[0] = '0';
-                return;
-            }
-
-            const bool is_negative = x < 0;
-            size_t i = 0;
-
-            // spare 1 byte for '\0'
-            while (x != 0 and i < m_buf.size() - 1)
-            {
-                const auto digit = std::labs(static_cast<long>(x % 10));
-                m_buf[i++] = static_cast<char>('0' + digit);
-                x /= 10;
-            }
-
-            // make sure the number has been processed completely
-            assert(x == 0);
-
-            if (is_negative)
-            {
-                // make sure there is capacity for the '-'
-                assert(i < m_buf.size() - 2);
-                m_buf[i++] = '-';
-            }
-
-            std::reverse(m_buf.begin(), m_buf.begin() + i);
-        }
-
-        template<typename NumberType>
-        void x_write(NumberType x, /*is_integral=*/std::false_type)
-        {
-            // special case for 0.0 and -0.0
-            if (x == 0)
-            {
-                size_t i = 0;
-                if (std::signbit(x))
-                {
-                    m_buf[i++] = '-';
-                }
-                m_buf[i++] = '0';
-                m_buf[i++] = '.';
-                m_buf[i] = '0';
-                return;
-            }
-
-            // get number of digits for a text -> float -> text round-trip
-            static constexpr auto d = std::numeric_limits<NumberType>::digits10;
-
-            // the actual conversion
-            const auto written_bytes = snprintf(m_buf.data(), m_buf.size(), "%.*g", d, x);
-
-            // negative value indicates an error
-            assert(written_bytes > 0);
-            // check if buffer was large enough
-            assert(static_cast<size_t>(written_bytes) < m_buf.size());
-
-            // read information from locale
-            const auto loc = localeconv();
-            assert(loc != nullptr);
-            const char thousands_sep = !loc->thousands_sep ? '\0'
-                                       : loc->thousands_sep[0];
-
-            const char decimal_point = !loc->decimal_point ? '\0'
-                                       : loc->decimal_point[0];
-
-            // erase thousands separator
-            if (thousands_sep != '\0')
-            {
-                const auto end = std::remove(m_buf.begin(), m_buf.begin() + written_bytes, thousands_sep);
-                std::fill(end, m_buf.end(), '\0');
-            }
-
-            // convert decimal point to '.'
-            if (decimal_point != '\0' and decimal_point != '.')
-            {
-                for (auto& c : m_buf)
-                {
-                    if (c == decimal_point)
-                    {
-                        c = '.';
-                        break;
-                    }
-                }
-            }
-
-            // determine if need to append ".0"
-            size_t i = 0;
-            bool value_is_int_like = true;
-            for (i = 0; i < m_buf.size(); ++i)
-            {
-                // break when end of number is reached
-                if (m_buf[i] == '\0')
-                {
-                    break;
-                }
-
-                // check if we find non-int character
-                value_is_int_like = value_is_int_like and m_buf[i] != '.' and
-                                    m_buf[i] != 'e' and m_buf[i] != 'E';
-            }
-
-            if (value_is_int_like)
-            {
-                // there must be 2 bytes left for ".0"
-                assert((i + 2) < m_buf.size());
-                // we write to the end of the number
-                assert(m_buf[i] == '\0');
-                assert(m_buf[i - 1] != '\0');
-
-                // add ".0"
-                m_buf[i] = '.';
-                m_buf[i + 1] = '0';
-
-                // the resulting string is properly terminated
-                assert(m_buf[i + 2] == '\0');
-            }
-        }
-    };
-
-
-    /*!
-    @brief internal implementation of the serialization function
-
-    This function is called by the public member function dump and organizes
-    the serialization internally. The indentation level is propagated as
-    additional parameter. In case of arrays and objects, the function is
-    called recursively. Note that
-
-    - strings and object keys are escaped using `escape_string()`
-    - integer numbers are converted implicitly via `operator<<`
-    - floating-point numbers are converted to a string using `"%g"` format
-
-    @param[out] o              stream to write to
-    @param[in] pretty_print    whether the output shall be pretty-printed
-    @param[in] indent_step     the indent level
-    @param[in] current_indent  the current indent level (only used internally)
-    */
-    void dump(std::ostream& o,
-              const bool pretty_print,
-              const unsigned int indent_step,
-              const unsigned int current_indent = 0) const
-    {
-        // variable to hold indentation for recursive calls
-        unsigned int new_indent = current_indent;
-
-        switch (m_type)
-        {
-            case value_t::object:
-            {
-                if (m_value.object->empty())
-                {
-                    o << "{}";
-                    return;
-                }
-
-                o << "{";
-
-                // increase indentation
-                if (pretty_print)
-                {
-                    new_indent += indent_step;
-                    o << "\n";
-                }
-
-                for (auto i = m_value.object->cbegin(); i != m_value.object->cend(); ++i)
-                {
-                    if (i != m_value.object->cbegin())
-                    {
-                        o << (pretty_print ? ",\n" : ",");
-                    }
-                    o << string_t(new_indent, ' ') << "\""
-                      << escape_string(i->first) << "\":"
-                      << (pretty_print ? " " : "");
-                    i->second.dump(o, pretty_print, indent_step, new_indent);
-                }
-
-                // decrease indentation
-                if (pretty_print)
-                {
-                    new_indent -= indent_step;
-                    o << "\n";
-                }
-
-                o << string_t(new_indent, ' ') + "}";
-                return;
-            }
-
-            case value_t::array:
-            {
-                if (m_value.array->empty())
-                {
-                    o << "[]";
-                    return;
-                }
-
-                o << "[";
-
-                // increase indentation
-                if (pretty_print)
-                {
-                    new_indent += indent_step;
-                    o << "\n";
-                }
-
-                for (auto i = m_value.array->cbegin(); i != m_value.array->cend(); ++i)
-                {
-                    if (i != m_value.array->cbegin())
-                    {
-                        o << (pretty_print ? ",\n" : ",");
-                    }
-                    o << string_t(new_indent, ' ');
-                    i->dump(o, pretty_print, indent_step, new_indent);
-                }
-
-                // decrease indentation
-                if (pretty_print)
-                {
-                    new_indent -= indent_step;
-                    o << "\n";
-                }
-
-                o << string_t(new_indent, ' ') << "]";
-                return;
-            }
-
-            case value_t::string:
-            {
-                o << string_t("\"") << escape_string(*m_value.string) << "\"";
-                return;
-            }
-
-            case value_t::boolean:
-            {
-                o << (m_value.boolean ? "true" : "false");
-                return;
-            }
-
-            case value_t::number_integer:
-            {
-                o << numtostr(m_value.number_integer).c_str();
-                return;
-            }
-
-            case value_t::number_unsigned:
-            {
-                o << numtostr(m_value.number_unsigned).c_str();
-                return;
-            }
-
-            case value_t::number_float:
-            {
-                o << numtostr(m_value.number_float).c_str();
-                return;
-            }
-
-            case value_t::discarded:
-            {
-                o << "<discarded>";
-                return;
-            }
-
-            case value_t::null:
-            {
-                o << "null";
-                return;
-            }
-        }
-    }
 
   private:
     //////////////////////
@@ -8896,50 +8186,35 @@ class basic_json
             }
         }
 
-        /*
-        Use operator `const_iterator` instead of `const_iterator(const iterator&
-        other) noexcept` to avoid two class definitions for @ref iterator and
-        @ref const_iterator.
-
-        This function is only called if this class is an @ref iterator. If this
-        class is a @ref const_iterator this function is not called.
+        /*!
+        @note The conventional copy constructor and copy assignment are
+              implicitly defined.
+              Combined with the following converting constructor and assigment,
+              they support: copy from iterator to iterator,
+                            copy from const iterator to const iterator,
+                            and conversion from iterator to const iterator.
+              However conversion from const iterator to iterator is not defined.
         */
-        operator const_iterator() const
-        {
-            const_iterator ret;
-
-            if (m_object)
-            {
-                ret.m_object = m_object;
-                ret.m_it = m_it;
-            }
-
-            return ret;
-        }
 
         /*!
-        @brief copy constructor
-        @param[in] other  iterator to copy from
+        @brief converting constructor
+        @param[in] other  non-const iterator to copy from
         @note It is not checked whether @a other is initialized.
         */
-        iter_impl(const iter_impl& other) noexcept
+        iter_impl(const iter_impl<basic_json>& other) noexcept
             : m_object(other.m_object), m_it(other.m_it)
         {}
 
         /*!
-        @brief copy assignment
-        @param[in,out] other  iterator to copy from
+        @brief converting assignment
+        @param[in,out] other  non-const iterator to copy from
+        @return const/non-const iterator
         @note It is not checked whether @a other is initialized.
         */
-        iter_impl& operator=(iter_impl other) noexcept(
-            std::is_nothrow_move_constructible<pointer>::value and
-            std::is_nothrow_move_assignable<pointer>::value and
-            std::is_nothrow_move_constructible<internal_iterator>::value and
-            std::is_nothrow_move_assignable<internal_iterator>::value
-        )
+        iter_impl& operator=(const iter_impl<basic_json>& other) noexcept
         {
-            std::swap(m_object, other.m_object);
-            std::swap(m_it, other.m_it);
+            m_object = other.m_object;
+            m_it = other.m_it;
             return *this;
         }
 
@@ -9036,7 +8311,7 @@ class basic_json
 
                 case basic_json::value_t::null:
                 {
-                    JSON_THROW(std::out_of_range("cannot get value"));
+                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
                 }
 
                 default:
@@ -9046,7 +8321,7 @@ class basic_json
                         return *m_object;
                     }
 
-                    JSON_THROW(std::out_of_range("cannot get value"));
+                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
                 }
             }
         }
@@ -9080,7 +8355,7 @@ class basic_json
                         return m_object;
                     }
 
-                    JSON_THROW(std::out_of_range("cannot get value"));
+                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
                 }
             }
         }
@@ -9180,7 +8455,7 @@ class basic_json
             // if objects are not the same, the comparison is undefined
             if (m_object != other.m_object)
             {
-                JSON_THROW(std::domain_error("cannot compare iterators of different containers"));
+                JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
             }
 
             assert(m_object != nullptr);
@@ -9222,7 +8497,7 @@ class basic_json
             // if objects are not the same, the comparison is undefined
             if (m_object != other.m_object)
             {
-                JSON_THROW(std::domain_error("cannot compare iterators of different containers"));
+                JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
             }
 
             assert(m_object != nullptr);
@@ -9231,7 +8506,7 @@ class basic_json
             {
                 case basic_json::value_t::object:
                 {
-                    JSON_THROW(std::domain_error("cannot compare order of object iterators"));
+                    JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators"));
                 }
 
                 case basic_json::value_t::array:
@@ -9285,7 +8560,7 @@ class basic_json
             {
                 case basic_json::value_t::object:
                 {
-                    JSON_THROW(std::domain_error("cannot use offsets with object iterators"));
+                    JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
                 }
 
                 case basic_json::value_t::array:
@@ -9317,18 +8592,29 @@ class basic_json
         @brief  add to iterator
         @pre The iterator is initialized; i.e. `m_object != nullptr`.
         */
-        iter_impl operator+(difference_type i)
+        iter_impl operator+(difference_type i) const
         {
             auto result = *this;
             result += i;
             return result;
         }
 
+        /*!
+        @brief  addition of distance and iterator
+        @pre The iterator is initialized; i.e. `m_object != nullptr`.
+        */
+        friend iter_impl operator+(difference_type i, const iter_impl& it)
+        {
+            auto result = it;
+            result += i;
+            return result;
+        }
+
         /*!
         @brief  subtract from iterator
         @pre The iterator is initialized; i.e. `m_object != nullptr`.
         */
-        iter_impl operator-(difference_type i)
+        iter_impl operator-(difference_type i) const
         {
             auto result = *this;
             result -= i;
@@ -9347,7 +8633,7 @@ class basic_json
             {
                 case basic_json::value_t::object:
                 {
-                    JSON_THROW(std::domain_error("cannot use offsets with object iterators"));
+                    JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
                 }
 
                 case basic_json::value_t::array:
@@ -9374,7 +8660,7 @@ class basic_json
             {
                 case basic_json::value_t::object:
                 {
-                    JSON_THROW(std::domain_error("cannot use operator[] for object iterators"));
+                    JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators"));
                 }
 
                 case basic_json::value_t::array:
@@ -9384,7 +8670,7 @@ class basic_json
 
                 case basic_json::value_t::null:
                 {
-                    JSON_THROW(std::out_of_range("cannot get value"));
+                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
                 }
 
                 default:
@@ -9394,7 +8680,7 @@ class basic_json
                         return *m_object;
                     }
 
-                    JSON_THROW(std::out_of_range("cannot get value"));
+                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
                 }
             }
         }
@@ -9412,7 +8698,7 @@ class basic_json
                 return m_it.object_iterator->first;
             }
 
-            JSON_THROW(std::domain_error("cannot use key() for non-object iterators"));
+            JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators"));
         }
 
         /*!
@@ -9428,7 +8714,7 @@ class basic_json
         /// associated JSON instance
         pointer m_object = nullptr;
         /// the actual iterator of the associated instance
-        internal_iterator m_it = internal_iterator();
+        struct internal_iterator m_it = internal_iterator();
     };
 
     /*!
@@ -9470,56 +8756,49 @@ class basic_json
         /// post-increment (it++)
         json_reverse_iterator operator++(int)
         {
-            return base_iterator::operator++(1);
+            return static_cast<json_reverse_iterator>(base_iterator::operator++(1));
         }
 
         /// pre-increment (++it)
         json_reverse_iterator& operator++()
         {
-            base_iterator::operator++();
-            return *this;
+            return static_cast<json_reverse_iterator&>(base_iterator::operator++());
         }
 
         /// post-decrement (it--)
         json_reverse_iterator operator--(int)
         {
-            return base_iterator::operator--(1);
+            return static_cast<json_reverse_iterator>(base_iterator::operator--(1));
         }
 
         /// pre-decrement (--it)
         json_reverse_iterator& operator--()
         {
-            base_iterator::operator--();
-            return *this;
+            return static_cast<json_reverse_iterator&>(base_iterator::operator--());
         }
 
         /// add to iterator
         json_reverse_iterator& operator+=(difference_type i)
         {
-            base_iterator::operator+=(i);
-            return *this;
+            return static_cast<json_reverse_iterator&>(base_iterator::operator+=(i));
         }
 
         /// add to iterator
         json_reverse_iterator operator+(difference_type i) const
         {
-            auto result = *this;
-            result += i;
-            return result;
+            return static_cast<json_reverse_iterator>(base_iterator::operator+(i));
         }
 
         /// subtract from iterator
         json_reverse_iterator operator-(difference_type i) const
         {
-            auto result = *this;
-            result -= i;
-            return result;
+            return static_cast<json_reverse_iterator>(base_iterator::operator-(i));
         }
 
         /// return difference
         difference_type operator-(const json_reverse_iterator& other) const
         {
-            return this->base() - other.base();
+            return base_iterator(*this) - base_iterator(other);
         }
 
         /// access to successor
@@ -9545,16 +8824,2292 @@ class basic_json
 
 
   private:
+    ////////////////////
+    // input adapters //
+    ////////////////////
+
+    /// abstract input adapter interface
+    class input_adapter
+    {
+      public:
+        virtual int get_character() = 0;
+        virtual std::string read(size_t offset, size_t length) = 0;
+        virtual ~input_adapter() {}
+
+        // native support
+
+        /// input adapter for input stream
+        static std::shared_ptr<input_adapter> create(std::istream& i)
+        {
+            return std::shared_ptr<input_adapter>(new cached_input_stream_adapter<16384>(i));
+        }
+
+        /// input adapter for input stream
+        static std::shared_ptr<input_adapter> create(std::istream&& i)
+        {
+            return std::shared_ptr<input_adapter>(new cached_input_stream_adapter<16384>(i));
+        }
+
+        /// input adapter for buffer
+        static std::shared_ptr<input_adapter> create(const char* b, size_t l)
+        {
+            return std::shared_ptr<input_adapter>(new input_buffer_adapter(b, l));
+        }
+
+        // derived support
+
+        /// input adapter for string literal
+        template<typename CharT, typename std::enable_if<
+                     std::is_pointer<CharT>::value and
+                     std::is_integral<typename std::remove_pointer<CharT>::type>::value and
+                     sizeof(typename std::remove_pointer<CharT>::type) == 1, int>::type = 0>
+        static std::shared_ptr<input_adapter> create(CharT b)
+        {
+            return create(reinterpret_cast<const char*>(b),
+                          std::strlen(reinterpret_cast<const char*>(b)));
+        }
+
+        /// input adapter for iterator range with contiguous storage
+        template<class IteratorType, typename std::enable_if<
+                     std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value
+                     , int>::type
+                 = 0>
+        static std::shared_ptr<input_adapter> create(IteratorType first, IteratorType last)
+        {
+            // assertion to check that the iterator range is indeed contiguous,
+            // see http://stackoverflow.com/a/35008842/266378 for more discussion
+            assert(std::accumulate(first, last, std::pair<bool, int>(true, 0),
+                                   [&first](std::pair<bool, int> res, decltype(*first) val)
+            {
+                res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
+                return res;
+            }).first);
+
+            // assertion to check that each element is 1 byte long
+            static_assert(sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
+                          "each element in the iterator range must have the size of 1 byte");
+
+            return create(reinterpret_cast<const char*>(&(*first)),
+                          static_cast<size_t>(std::distance(first, last)));
+        }
+
+        /// input adapter for array
+        template<class T, std::size_t N>
+        static std::shared_ptr<input_adapter> create(T (&array)[N])
+        {
+            // delegate the call to the iterator-range overload
+            return create(std::begin(array), std::end(array));
+        }
+
+        /// input adapter for contiguous container
+        template<class ContiguousContainer, typename std::enable_if<
+                     not std::is_pointer<ContiguousContainer>::value and
+                     std::is_base_of<
+                         std::random_access_iterator_tag,
+                         typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value
+                     , int>::type = 0>
+        static std::shared_ptr<input_adapter> create(const ContiguousContainer& c)
+        {
+            // delegate the call to the iterator-range overload
+            return create(std::begin(c), std::end(c));
+        }
+    };
+
+    /// a type to simplify interfaces
+    using input_adapter_t = std::shared_ptr<input_adapter>;
+
+    /// input adapter for cached stream input
+    template<std::size_t N>
+    class cached_input_stream_adapter : public input_adapter
+    {
+      public:
+        cached_input_stream_adapter(std::istream& i)
+            : is(i), start_position(is.tellg())
+        {
+            // immediately abort if stream is erroneous
+            if (JSON_UNLIKELY(i.fail()))
+            {
+                JSON_THROW(parse_error::create(111, 0, "bad input stream"));
+            }
+
+            fill_buffer();
+
+            // skip byte order mark
+            if (fill_size >= 3 and buffer[0] == '\xEF' and buffer[1] == '\xBB' and buffer[2] == '\xBF')
+            {
+                buffer_pos += 3;
+                processed_chars += 3;
+            }
+        }
+
+        ~cached_input_stream_adapter() override
+        {
+            // clear stream flags
+            is.clear();
+            // We initially read a lot of characters into the buffer, and we
+            // may not have processed all of them. Therefore, we need to
+            // "rewind" the stream after the last processed char.
+            is.seekg(start_position);
+            is.ignore(static_cast<std::streamsize>(processed_chars));
+            // clear stream flags
+            is.clear();
+        }
+
+        int get_character() override
+        {
+            // check if refilling is necessary and possible
+            if (buffer_pos == fill_size and not eof)
+            {
+                fill_buffer();
+
+                // check and remember that filling did not yield new input
+                if (fill_size == 0)
+                {
+                    eof = true;
+                    return std::char_traits<char>::eof();
+                }
+
+                // the buffer is ready
+                buffer_pos = 0;
+            }
+
+            ++processed_chars;
+            assert(buffer_pos < buffer.size());
+            return buffer[buffer_pos++] & 0xFF;
+        }
+
+        std::string read(size_t offset, size_t length) override
+        {
+            // create buffer
+            std::string result(length, '\0');
+
+            // save stream position
+            const auto current_pos = is.tellg();
+            // save stream flags
+            const auto flags = is.rdstate();
+
+            // clear stream flags
+            is.clear();
+            // set stream position
+            is.seekg(static_cast<std::streamoff>(offset));
+            // read bytes
+            is.read(&result[0], static_cast<std::streamsize>(length));
+
+            // reset stream position
+            is.seekg(current_pos);
+            // reset stream flags
+            is.setstate(flags);
+
+            return result;
+        }
+
+      private:
+        void fill_buffer()
+        {
+            // fill
+            is.read(buffer.data(), static_cast<std::streamsize>(buffer.size()));
+            // store number of bytes in the buffer
+            fill_size = static_cast<size_t>(is.gcount());
+        }
+
+        /// the associated input stream
+        std::istream& is;
+
+        /// chars returned via get_character()
+        size_t processed_chars = 0;
+        /// chars processed in the current buffer
+        size_t buffer_pos = 0;
+
+        /// whether stream reached eof
+        bool eof = false;
+        /// how many chars have been copied to the buffer by last (re)fill
+        size_t fill_size = 0;
+
+        /// position of the stream when we started
+        const std::streampos start_position;
+
+        /// internal buffer
+        std::array<char, N> buffer{{}};
+    };
+
+    /// input adapter for buffer input
+    class input_buffer_adapter : public input_adapter
+    {
+      public:
+        input_buffer_adapter(const char* b, size_t l)
+            : input_adapter(), cursor(b), limit(b + l), start(b)
+        {
+            // skip byte order mark
+            if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF')
+            {
+                cursor += 3;
+            }
+        }
+
+        // delete because of pointer members
+        input_buffer_adapter(const input_buffer_adapter&) = delete;
+        input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
+
+        int get_character() noexcept override
+        {
+            if (JSON_LIKELY(cursor < limit))
+            {
+                return *(cursor++) & 0xFF;
+            }
+            else
+            {
+                return std::char_traits<char>::eof();
+            }
+        }
+
+        std::string read(size_t offset, size_t length) override
+        {
+            // avoid reading too many characters
+            const size_t max_length = static_cast<size_t>(limit - start);
+            return std::string(start + offset, (std::min)(length, max_length - offset));
+        }
+
+      private:
+        /// pointer to the current character
+        const char* cursor;
+        /// pointer past the last character
+        const char* limit;
+        /// pointer to the first character
+        const char* start;
+    };
+
+    //////////////////////////////////////////
+    // binary serialization/deserialization //
+    //////////////////////////////////////////
+
+    /// @name binary serialization/deserialization support
+    /// @{
+
+  private:
+    /*!
+    @brief deserialization of CBOR and MessagePack values
+    */
+    class binary_reader
+    {
+      public:
+        /*!
+        @brief create a binary reader
+
+        @param[in] adapter  input adapter to read from
+        */
+        explicit binary_reader(input_adapter_t adapter)
+            : ia(adapter), is_little_endian(little_endianess())
+        {
+            assert(ia);
+        }
+
+        /*!
+        @brief create a JSON value from CBOR input
+
+        @param[in] get_char  whether a new character should be retrieved from
+                             the input (true, default) or whether the last
+                             read character should be considered instead
+
+        @return JSON value created from CBOR input
+
+        @throw parse_error.110 if input ended unexpectedly
+        @throw parse_error.112 if unsupported byte was read
+        */
+        basic_json parse_cbor(const bool get_char = true)
+        {
+            switch (get_char ? get() : current)
+            {
+                // EOF
+                case std::char_traits<char>::eof():
+                {
+                    JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+                }
+
+                // Integer 0x00..0x17 (0..23)
+                case 0x00:
+                case 0x01:
+                case 0x02:
+                case 0x03:
+                case 0x04:
+                case 0x05:
+                case 0x06:
+                case 0x07:
+                case 0x08:
+                case 0x09:
+                case 0x0a:
+                case 0x0b:
+                case 0x0c:
+                case 0x0d:
+                case 0x0e:
+                case 0x0f:
+                case 0x10:
+                case 0x11:
+                case 0x12:
+                case 0x13:
+                case 0x14:
+                case 0x15:
+                case 0x16:
+                case 0x17:
+                {
+                    return static_cast<number_unsigned_t>(current);
+                }
+
+                case 0x18: // Unsigned integer (one-byte uint8_t follows)
+                {
+                    return get_number<uint8_t>();
+                }
+
+                case 0x19: // Unsigned integer (two-byte uint16_t follows)
+                {
+                    return get_number<uint16_t>();
+                }
+
+                case 0x1a: // Unsigned integer (four-byte uint32_t follows)
+                {
+                    return get_number<uint32_t>();
+                }
+
+                case 0x1b: // Unsigned integer (eight-byte uint64_t follows)
+                {
+                    return get_number<uint64_t>();
+                }
+
+                // Negative integer -1-0x00..-1-0x17 (-1..-24)
+                case 0x20:
+                case 0x21:
+                case 0x22:
+                case 0x23:
+                case 0x24:
+                case 0x25:
+                case 0x26:
+                case 0x27:
+                case 0x28:
+                case 0x29:
+                case 0x2a:
+                case 0x2b:
+                case 0x2c:
+                case 0x2d:
+                case 0x2e:
+                case 0x2f:
+                case 0x30:
+                case 0x31:
+                case 0x32:
+                case 0x33:
+                case 0x34:
+                case 0x35:
+                case 0x36:
+                case 0x37:
+                {
+                    return static_cast<int8_t>(0x20 - 1 - current);
+                }
+
+                case 0x38: // Negative integer (one-byte uint8_t follows)
+                {
+                    // must be uint8_t !
+                    return static_cast<number_integer_t>(-1) - get_number<uint8_t>();
+                }
+
+                case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
+                {
+                    return static_cast<number_integer_t>(-1) - get_number<uint16_t>();
+                }
+
+                case 0x3a: // Negative integer -1-n (four-byte uint32_t follows)
+                {
+                    return static_cast<number_integer_t>(-1) - get_number<uint32_t>();
+                }
+
+                case 0x3b: // Negative integer -1-n (eight-byte uint64_t follows)
+                {
+                    return static_cast<number_integer_t>(-1) - static_cast<number_integer_t>(get_number<uint64_t>());
+                }
+
+                // UTF-8 string (0x00..0x17 bytes follow)
+                case 0x60:
+                case 0x61:
+                case 0x62:
+                case 0x63:
+                case 0x64:
+                case 0x65:
+                case 0x66:
+                case 0x67:
+                case 0x68:
+                case 0x69:
+                case 0x6a:
+                case 0x6b:
+                case 0x6c:
+                case 0x6d:
+                case 0x6e:
+                case 0x6f:
+                case 0x70:
+                case 0x71:
+                case 0x72:
+                case 0x73:
+                case 0x74:
+                case 0x75:
+                case 0x76:
+                case 0x77:
+                case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+                case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+                case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
+                case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
+                case 0x7f: // UTF-8 string (indefinite length)
+                {
+                    return get_cbor_string();
+                }
+
+                // array (0x00..0x17 data items follow)
+                case 0x80:
+                case 0x81:
+                case 0x82:
+                case 0x83:
+                case 0x84:
+                case 0x85:
+                case 0x86:
+                case 0x87:
+                case 0x88:
+                case 0x89:
+                case 0x8a:
+                case 0x8b:
+                case 0x8c:
+                case 0x8d:
+                case 0x8e:
+                case 0x8f:
+                case 0x90:
+                case 0x91:
+                case 0x92:
+                case 0x93:
+                case 0x94:
+                case 0x95:
+                case 0x96:
+                case 0x97:
+                {
+                    basic_json result = value_t::array;
+                    const auto len = static_cast<size_t>(current & 0x1f);
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        result.push_back(parse_cbor());
+                    }
+                    return result;
+                }
+
+                case 0x98: // array (one-byte uint8_t for n follows)
+                {
+                    basic_json result = value_t::array;
+                    const auto len = static_cast<size_t>(get_number<uint8_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        result.push_back(parse_cbor());
+                    }
+                    return result;
+                }
+
+                case 0x99: // array (two-byte uint16_t for n follow)
+                {
+                    basic_json result = value_t::array;
+                    const auto len = static_cast<size_t>(get_number<uint16_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        result.push_back(parse_cbor());
+                    }
+                    return result;
+                }
+
+                case 0x9a: // array (four-byte uint32_t for n follow)
+                {
+                    basic_json result = value_t::array;
+                    const auto len = static_cast<size_t>(get_number<uint32_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        result.push_back(parse_cbor());
+                    }
+                    return result;
+                }
+
+                case 0x9b: // array (eight-byte uint64_t for n follow)
+                {
+                    basic_json result = value_t::array;
+                    const auto len = static_cast<size_t>(get_number<uint64_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        result.push_back(parse_cbor());
+                    }
+                    return result;
+                }
+
+                case 0x9f: // array (indefinite length)
+                {
+                    basic_json result = value_t::array;
+                    while (get() != 0xff)
+                    {
+                        result.push_back(parse_cbor(false));
+                    }
+                    return result;
+                }
+
+                // map (0x00..0x17 pairs of data items follow)
+                case 0xa0:
+                case 0xa1:
+                case 0xa2:
+                case 0xa3:
+                case 0xa4:
+                case 0xa5:
+                case 0xa6:
+                case 0xa7:
+                case 0xa8:
+                case 0xa9:
+                case 0xaa:
+                case 0xab:
+                case 0xac:
+                case 0xad:
+                case 0xae:
+                case 0xaf:
+                case 0xb0:
+                case 0xb1:
+                case 0xb2:
+                case 0xb3:
+                case 0xb4:
+                case 0xb5:
+                case 0xb6:
+                case 0xb7:
+                {
+                    basic_json result = value_t::object;
+                    const auto len = static_cast<size_t>(current & 0x1f);
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        get();
+                        auto key = get_cbor_string();
+                        result[key] = parse_cbor();
+                    }
+                    return result;
+                }
+
+                case 0xb8: // map (one-byte uint8_t for n follows)
+                {
+                    basic_json result = value_t::object;
+                    const auto len = static_cast<size_t>(get_number<uint8_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        get();
+                        auto key = get_cbor_string();
+                        result[key] = parse_cbor();
+                    }
+                    return result;
+                }
+
+                case 0xb9: // map (two-byte uint16_t for n follow)
+                {
+                    basic_json result = value_t::object;
+                    const auto len = static_cast<size_t>(get_number<uint16_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        get();
+                        auto key = get_cbor_string();
+                        result[key] = parse_cbor();
+                    }
+                    return result;
+                }
+
+                case 0xba: // map (four-byte uint32_t for n follow)
+                {
+                    basic_json result = value_t::object;
+                    const auto len = static_cast<size_t>(get_number<uint32_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        get();
+                        auto key = get_cbor_string();
+                        result[key] = parse_cbor();
+                    }
+                    return result;
+                }
+
+                case 0xbb: // map (eight-byte uint64_t for n follow)
+                {
+                    basic_json result = value_t::object;
+                    const auto len = static_cast<size_t>(get_number<uint64_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        get();
+                        auto key = get_cbor_string();
+                        result[key] = parse_cbor();
+                    }
+                    return result;
+                }
+
+                case 0xbf: // map (indefinite length)
+                {
+                    basic_json result = value_t::object;
+                    while (get() != 0xff)
+                    {
+                        auto key = get_cbor_string();
+                        result[key] = parse_cbor();
+                    }
+                    return result;
+                }
+
+                case 0xf4: // false
+                {
+                    return false;
+                }
+
+                case 0xf5: // true
+                {
+                    return true;
+                }
+
+                case 0xf6: // null
+                {
+                    return value_t::null;
+                }
+
+                case 0xf9: // Half-Precision Float (two-byte IEEE 754)
+                {
+                    const int byte1 = get();
+                    check_eof();
+                    const int byte2 = get();
+                    check_eof();
+
+                    // code from RFC 7049, Appendix D, Figure 3:
+                    // As half-precision floating-point numbers were only added
+                    // to IEEE 754 in 2008, today's programming platforms often
+                    // still only have limited support for them. It is very
+                    // easy to include at least decoding support for them even
+                    // without such support. An example of a small decoder for
+                    // half-precision floating-point numbers in the C language
+                    // is shown in Fig. 3.
+                    const int half = (byte1 << 8) + byte2;
+                    const int exp = (half >> 10) & 0x1f;
+                    const int mant = half & 0x3ff;
+                    double val;
+                    if (exp == 0)
+                    {
+                        val = std::ldexp(mant, -24);
+                    }
+                    else if (exp != 31)
+                    {
+                        val = std::ldexp(mant + 1024, exp - 25);
+                    }
+                    else
+                    {
+                        val = mant == 0
+                              ? std::numeric_limits<double>::infinity()
+                              : std::numeric_limits<double>::quiet_NaN();
+                    }
+                    return (half & 0x8000) != 0 ? -val : val;
+                }
+
+                case 0xfa: // Single-Precision Float (four-byte IEEE 754)
+                {
+                    return get_number<float>();
+                }
+
+                case 0xfb: // Double-Precision Float (eight-byte IEEE 754)
+                {
+                    return get_number<double>();
+                }
+
+                default: // anything else (0xFF is handled inside the other types)
+                {
+                    std::stringstream ss;
+                    ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                    JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str()));
+                }
+            }
+        }
+
+        /*!
+        @brief create a JSON value from MessagePack input
+
+        @return JSON value created from MessagePack input
+
+        @throw parse_error.110 if input ended unexpectedly
+        @throw parse_error.112 if unsupported byte was read
+        */
+        basic_json parse_msgpack()
+        {
+            switch (get())
+            {
+                // EOF
+                case std::char_traits<char>::eof():
+                {
+                    JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+                }
+
+                // positive fixint
+                case 0x00:
+                case 0x01:
+                case 0x02:
+                case 0x03:
+                case 0x04:
+                case 0x05:
+                case 0x06:
+                case 0x07:
+                case 0x08:
+                case 0x09:
+                case 0x0a:
+                case 0x0b:
+                case 0x0c:
+                case 0x0d:
+                case 0x0e:
+                case 0x0f:
+                case 0x10:
+                case 0x11:
+                case 0x12:
+                case 0x13:
+                case 0x14:
+                case 0x15:
+                case 0x16:
+                case 0x17:
+                case 0x18:
+                case 0x19:
+                case 0x1a:
+                case 0x1b:
+                case 0x1c:
+                case 0x1d:
+                case 0x1e:
+                case 0x1f:
+                case 0x20:
+                case 0x21:
+                case 0x22:
+                case 0x23:
+                case 0x24:
+                case 0x25:
+                case 0x26:
+                case 0x27:
+                case 0x28:
+                case 0x29:
+                case 0x2a:
+                case 0x2b:
+                case 0x2c:
+                case 0x2d:
+                case 0x2e:
+                case 0x2f:
+                case 0x30:
+                case 0x31:
+                case 0x32:
+                case 0x33:
+                case 0x34:
+                case 0x35:
+                case 0x36:
+                case 0x37:
+                case 0x38:
+                case 0x39:
+                case 0x3a:
+                case 0x3b:
+                case 0x3c:
+                case 0x3d:
+                case 0x3e:
+                case 0x3f:
+                case 0x40:
+                case 0x41:
+                case 0x42:
+                case 0x43:
+                case 0x44:
+                case 0x45:
+                case 0x46:
+                case 0x47:
+                case 0x48:
+                case 0x49:
+                case 0x4a:
+                case 0x4b:
+                case 0x4c:
+                case 0x4d:
+                case 0x4e:
+                case 0x4f:
+                case 0x50:
+                case 0x51:
+                case 0x52:
+                case 0x53:
+                case 0x54:
+                case 0x55:
+                case 0x56:
+                case 0x57:
+                case 0x58:
+                case 0x59:
+                case 0x5a:
+                case 0x5b:
+                case 0x5c:
+                case 0x5d:
+                case 0x5e:
+                case 0x5f:
+                case 0x60:
+                case 0x61:
+                case 0x62:
+                case 0x63:
+                case 0x64:
+                case 0x65:
+                case 0x66:
+                case 0x67:
+                case 0x68:
+                case 0x69:
+                case 0x6a:
+                case 0x6b:
+                case 0x6c:
+                case 0x6d:
+                case 0x6e:
+                case 0x6f:
+                case 0x70:
+                case 0x71:
+                case 0x72:
+                case 0x73:
+                case 0x74:
+                case 0x75:
+                case 0x76:
+                case 0x77:
+                case 0x78:
+                case 0x79:
+                case 0x7a:
+                case 0x7b:
+                case 0x7c:
+                case 0x7d:
+                case 0x7e:
+                case 0x7f:
+                {
+                    return static_cast<number_unsigned_t>(current);
+                }
+
+                // fixmap
+                case 0x80:
+                case 0x81:
+                case 0x82:
+                case 0x83:
+                case 0x84:
+                case 0x85:
+                case 0x86:
+                case 0x87:
+                case 0x88:
+                case 0x89:
+                case 0x8a:
+                case 0x8b:
+                case 0x8c:
+                case 0x8d:
+                case 0x8e:
+                case 0x8f:
+                {
+                    basic_json result = value_t::object;
+                    const auto len = static_cast<size_t>(current & 0x0f);
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        get();
+                        auto key = get_msgpack_string();
+                        result[key] = parse_msgpack();
+                    }
+                    return result;
+                }
+
+                // fixarray
+                case 0x90:
+                case 0x91:
+                case 0x92:
+                case 0x93:
+                case 0x94:
+                case 0x95:
+                case 0x96:
+                case 0x97:
+                case 0x98:
+                case 0x99:
+                case 0x9a:
+                case 0x9b:
+                case 0x9c:
+                case 0x9d:
+                case 0x9e:
+                case 0x9f:
+                {
+                    basic_json result = value_t::array;
+                    const auto len = static_cast<size_t>(current & 0x0f);
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        result.push_back(parse_msgpack());
+                    }
+                    return result;
+                }
+
+                // fixstr
+                case 0xa0:
+                case 0xa1:
+                case 0xa2:
+                case 0xa3:
+                case 0xa4:
+                case 0xa5:
+                case 0xa6:
+                case 0xa7:
+                case 0xa8:
+                case 0xa9:
+                case 0xaa:
+                case 0xab:
+                case 0xac:
+                case 0xad:
+                case 0xae:
+                case 0xaf:
+                case 0xb0:
+                case 0xb1:
+                case 0xb2:
+                case 0xb3:
+                case 0xb4:
+                case 0xb5:
+                case 0xb6:
+                case 0xb7:
+                case 0xb8:
+                case 0xb9:
+                case 0xba:
+                case 0xbb:
+                case 0xbc:
+                case 0xbd:
+                case 0xbe:
+                case 0xbf:
+                {
+                    return get_msgpack_string();
+                }
+
+                case 0xc0: // nil
+                {
+                    return value_t::null;
+                }
+
+                case 0xc2: // false
+                {
+                    return false;
+                }
+
+                case 0xc3: // true
+                {
+                    return true;
+                }
+
+                case 0xca: // float 32
+                {
+                    return get_number<float>();
+                }
+
+                case 0xcb: // float 64
+                {
+                    return get_number<double>();
+                }
+
+                case 0xcc: // uint 8
+                {
+                    return get_number<uint8_t>();
+                }
+
+                case 0xcd: // uint 16
+                {
+                    return get_number<uint16_t>();
+                }
+
+                case 0xce: // uint 32
+                {
+                    return get_number<uint32_t>();
+                }
+
+                case 0xcf: // uint 64
+                {
+                    return get_number<uint64_t>();
+                }
+
+                case 0xd0: // int 8
+                {
+                    return get_number<int8_t>();
+                }
+
+                case 0xd1: // int 16
+                {
+                    return get_number<int16_t>();
+                }
+
+                case 0xd2: // int 32
+                {
+                    return get_number<int32_t>();
+                }
+
+                case 0xd3: // int 64
+                {
+                    return get_number<int64_t>();
+                }
+
+                case 0xd9: // str 8
+                case 0xda: // str 16
+                case 0xdb: // str 32
+                {
+                    return get_msgpack_string();
+                }
+
+                case 0xdc: // array 16
+                {
+                    basic_json result = value_t::array;
+                    const auto len = static_cast<size_t>(get_number<uint16_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        result.push_back(parse_msgpack());
+                    }
+                    return result;
+                }
+
+                case 0xdd: // array 32
+                {
+                    basic_json result = value_t::array;
+                    const auto len = static_cast<size_t>(get_number<uint32_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        result.push_back(parse_msgpack());
+                    }
+                    return result;
+                }
+
+                case 0xde: // map 16
+                {
+                    basic_json result = value_t::object;
+                    const auto len = static_cast<size_t>(get_number<uint16_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        get();
+                        auto key = get_msgpack_string();
+                        result[key] = parse_msgpack();
+                    }
+                    return result;
+                }
+
+                case 0xdf: // map 32
+                {
+                    basic_json result = value_t::object;
+                    const auto len = static_cast<size_t>(get_number<uint32_t>());
+                    for (size_t i = 0; i < len; ++i)
+                    {
+                        get();
+                        auto key = get_msgpack_string();
+                        result[key] = parse_msgpack();
+                    }
+                    return result;
+                }
+
+                // positive fixint
+                case 0xe0:
+                case 0xe1:
+                case 0xe2:
+                case 0xe3:
+                case 0xe4:
+                case 0xe5:
+                case 0xe6:
+                case 0xe7:
+                case 0xe8:
+                case 0xe9:
+                case 0xea:
+                case 0xeb:
+                case 0xec:
+                case 0xed:
+                case 0xee:
+                case 0xef:
+                case 0xf0:
+                case 0xf1:
+                case 0xf2:
+                case 0xf3:
+                case 0xf4:
+                case 0xf5:
+                case 0xf6:
+                case 0xf7:
+                case 0xf8:
+                case 0xf9:
+                case 0xfa:
+                case 0xfb:
+                case 0xfc:
+                case 0xfd:
+                case 0xfe:
+                case 0xff:
+                {
+                    return static_cast<int8_t>(current);
+                }
+
+                default: // anything else
+                {
+                    std::stringstream ss;
+                    ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                    JSON_THROW(parse_error::create(112, chars_read, "error reading MessagePack; last byte: 0x" + ss.str()));
+                }
+            }
+        }
+
+        /*!
+        @brief determine system byte order
+
+        @return true iff system's byte order is little endian
+
+        @note from http://stackoverflow.com/a/1001328/266378
+        */
+        static bool little_endianess() noexcept
+        {
+            int num = 1;
+            return (*reinterpret_cast<char*>(&num) == 1);
+        }
+
+      private:
+        /*!
+        @brief get next character from the input
+
+        This function provides the interface to the used input adapter. It does
+        not throw in case the input reached EOF, but returns
+        `std::char_traits<char>::eof()` in that case.
+
+        @return character read from the input
+        */
+        int get()
+        {
+            ++chars_read;
+            return (current = ia->get_character());
+        }
+
+        /*
+        @brief read a number from the input
+
+        @tparam T the type of the number
+
+        @return number of type @a T
+
+        @note This function needs to respect the system's endianess, because
+              bytes in CBOR and MessagePack are stored in network order (big
+              endian) and therefore need reordering on little endian systems.
+
+        @throw parse_error.110 if input has less than `sizeof(T)` bytes
+        */
+        template<typename T>
+        T get_number()
+        {
+            // step 1: read input into array with system's byte order
+            std::array<uint8_t, sizeof(T)> vec;
+            for (size_t i = 0; i < sizeof(T); ++i)
+            {
+                get();
+                check_eof();
+
+                // reverse byte order prior to conversion if necessary
+                if (is_little_endian)
+                {
+                    vec[sizeof(T) - i - 1] = static_cast<uint8_t>(current);
+                }
+                else
+                {
+                    vec[i] = static_cast<uint8_t>(current);  // LCOV_EXCL_LINE
+                }
+            }
+
+            // step 2: convert array into number of type T and return
+            T result;
+            std::memcpy(&result, vec.data(), sizeof(T));
+            return result;
+        }
+
+        /*!
+        @brief create a string by reading characters from the input
+
+        @param[in] len number of bytes to read
+
+        @return string created by reading @a len bytes
+
+        @throw parse_error.110 if input has less than @a len bytes
+        */
+        std::string get_string(const size_t len)
+        {
+            std::string result;
+            for (size_t i = 0; i < len; ++i)
+            {
+                get();
+                check_eof();
+                result.append(1, static_cast<char>(current));
+            }
+            return result;
+        }
+
+        /*!
+        @brief reads a CBOR string
+
+        This function first reads starting bytes to determine the expected
+        string length and then copies this number of bytes into a string.
+        Additionally, CBOR's strings with indefinite lengths are supported.
+
+        @return string
+
+        @throw parse_error.110 if input ended
+        @throw parse_error.113 if an unexpexted byte is read
+        */
+        std::string get_cbor_string()
+        {
+            check_eof();
+
+            switch (current)
+            {
+                // UTF-8 string (0x00..0x17 bytes follow)
+                case 0x60:
+                case 0x61:
+                case 0x62:
+                case 0x63:
+                case 0x64:
+                case 0x65:
+                case 0x66:
+                case 0x67:
+                case 0x68:
+                case 0x69:
+                case 0x6a:
+                case 0x6b:
+                case 0x6c:
+                case 0x6d:
+                case 0x6e:
+                case 0x6f:
+                case 0x70:
+                case 0x71:
+                case 0x72:
+                case 0x73:
+                case 0x74:
+                case 0x75:
+                case 0x76:
+                case 0x77:
+                {
+                    const auto len = static_cast<size_t>(current & 0x1f);
+                    return get_string(len);
+                }
+
+                case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+                {
+                    const auto len = static_cast<size_t>(get_number<uint8_t>());
+                    return get_string(len);
+                }
+
+                case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+                {
+                    const auto len = static_cast<size_t>(get_number<uint16_t>());
+                    return get_string(len);
+                }
+
+                case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
+                {
+                    const auto len = static_cast<size_t>(get_number<uint32_t>());
+                    return get_string(len);
+                }
+
+                case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
+                {
+                    const auto len = static_cast<size_t>(get_number<uint64_t>());
+                    return get_string(len);
+                }
+
+                case 0x7f: // UTF-8 string (indefinite length)
+                {
+                    std::string result;
+                    while (get() != 0xff)
+                    {
+                        check_eof();
+                        result.append(1, static_cast<char>(current));
+                    }
+                    return result;
+                }
+
+                default:
+                {
+                    std::stringstream ss;
+                    ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                    JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str()));
+                }
+            }
+        }
+
+        /*!
+        @brief reads a MessagePack string
+
+        This function first reads starting bytes to determine the expected
+        string length and then copies this number of bytes into a string.
+
+        @return string
+
+        @throw parse_error.110 if input ended
+        @throw parse_error.113 if an unexpexted byte is read
+        */
+        std::string get_msgpack_string()
+        {
+            check_eof();
+
+            switch (current)
+            {
+                // fixstr
+                case 0xa0:
+                case 0xa1:
+                case 0xa2:
+                case 0xa3:
+                case 0xa4:
+                case 0xa5:
+                case 0xa6:
+                case 0xa7:
+                case 0xa8:
+                case 0xa9:
+                case 0xaa:
+                case 0xab:
+                case 0xac:
+                case 0xad:
+                case 0xae:
+                case 0xaf:
+                case 0xb0:
+                case 0xb1:
+                case 0xb2:
+                case 0xb3:
+                case 0xb4:
+                case 0xb5:
+                case 0xb6:
+                case 0xb7:
+                case 0xb8:
+                case 0xb9:
+                case 0xba:
+                case 0xbb:
+                case 0xbc:
+                case 0xbd:
+                case 0xbe:
+                case 0xbf:
+                {
+                    const auto len = static_cast<size_t>(current & 0x1f);
+                    return get_string(len);
+                }
+
+                case 0xd9: // str 8
+                {
+                    const auto len = static_cast<size_t>(get_number<uint8_t>());
+                    return get_string(len);
+                }
+
+                case 0xda: // str 16
+                {
+                    const auto len = static_cast<size_t>(get_number<uint16_t>());
+                    return get_string(len);
+                }
+
+                case 0xdb: // str 32
+                {
+                    const auto len = static_cast<size_t>(get_number<uint32_t>());
+                    return get_string(len);
+                }
+
+                default:
+                {
+                    std::stringstream ss;
+                    ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                    JSON_THROW(parse_error::create(113, chars_read, "expected a MessagePack string; last byte: 0x" + ss.str()));
+                }
+            }
+        }
+
+        /*!
+        @brief check if input ended
+        @throw parse_error.110 if input ended
+        */
+        void check_eof() const
+        {
+            if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
+            {
+                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+            }
+        }
+
+      private:
+        /// input adapter
+        input_adapter_t ia = nullptr;
+
+        /// the current character
+        int current = std::char_traits<char>::eof();
+
+        /// the number of characters read
+        size_t chars_read = 0;
+
+        /// whether we can assume little endianess
+        const bool is_little_endian = true;
+    };
+
+    /*!
+    @brief serialization to CBOR and MessagePack values
+    */
+    class binary_writer
+    {
+      public:
+        /*!
+        @brief create a binary writer
+
+        @param[in] adapter  output adapter to write to
+        */
+        explicit binary_writer(output_adapter_t<uint8_t> adapter)
+            : is_little_endian(binary_reader::little_endianess()), oa(adapter)
+        {
+            assert(oa);
+        }
+
+        /*!
+        @brief[in] j  JSON value to serialize
+        */
+        void write_cbor(const basic_json& j)
+        {
+            switch (j.type())
+            {
+                case value_t::null:
+                {
+                    oa->write_character(0xf6);
+                    break;
+                }
+
+                case value_t::boolean:
+                {
+                    oa->write_character(j.m_value.boolean ? 0xf5 : 0xf4);
+                    break;
+                }
+
+                case value_t::number_integer:
+                {
+                    if (j.m_value.number_integer >= 0)
+                    {
+                        // CBOR does not differentiate between positive signed
+                        // integers and unsigned integers. Therefore, we used the
+                        // code from the value_t::number_unsigned case here.
+                        if (j.m_value.number_integer <= 0x17)
+                        {
+                            write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
+                        {
+                            oa->write_character(0x18);
+                            write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)())
+                        {
+                            oa->write_character(0x19);
+                            write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
+                        {
+                            oa->write_character(0x1a);
+                            write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                        }
+                        else
+                        {
+                            oa->write_character(0x1b);
+                            write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                        }
+                    }
+                    else
+                    {
+                        // The conversions below encode the sign in the first
+                        // byte, and the value is converted to a positive number.
+                        const auto positive_number = -1 - j.m_value.number_integer;
+                        if (j.m_value.number_integer >= -24)
+                        {
+                            write_number(static_cast<uint8_t>(0x20 + positive_number));
+                        }
+                        else if (positive_number <= (std::numeric_limits<uint8_t>::max)())
+                        {
+                            oa->write_character(0x38);
+                            write_number(static_cast<uint8_t>(positive_number));
+                        }
+                        else if (positive_number <= (std::numeric_limits<uint16_t>::max)())
+                        {
+                            oa->write_character(0x39);
+                            write_number(static_cast<uint16_t>(positive_number));
+                        }
+                        else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
+                        {
+                            oa->write_character(0x3a);
+                            write_number(static_cast<uint32_t>(positive_number));
+                        }
+                        else
+                        {
+                            oa->write_character(0x3b);
+                            write_number(static_cast<uint64_t>(positive_number));
+                        }
+                    }
+                    break;
+                }
+
+                case value_t::number_unsigned:
+                {
+                    if (j.m_value.number_unsigned <= 0x17)
+                    {
+                        write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        oa->write_character(0x18);
+                        write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        oa->write_character(0x19);
+                        write_number(static_cast<uint16_t>(j.m_value.number_unsigned));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        oa->write_character(0x1a);
+                        write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
+                    }
+                    else
+                    {
+                        oa->write_character(0x1b);
+                        write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
+                    }
+                    break;
+                }
+
+                case value_t::number_float:
+                {
+                    // Double-Precision Float
+                    oa->write_character(0xfb);
+                    write_number(j.m_value.number_float);
+                    break;
+                }
+
+                case value_t::string:
+                {
+                    // step 1: write control byte and the string length
+                    const auto N = j.m_value.string->size();
+                    if (N <= 0x17)
+                    {
+                        write_number(static_cast<uint8_t>(0x60 + N));
+                    }
+                    else if (N <= 0xff)
+                    {
+                        oa->write_character(0x78);
+                        write_number(static_cast<uint8_t>(N));
+                    }
+                    else if (N <= 0xffff)
+                    {
+                        oa->write_character(0x79);
+                        write_number(static_cast<uint16_t>(N));
+                    }
+                    else if (N <= 0xffffffff)
+                    {
+                        oa->write_character(0x7a);
+                        write_number(static_cast<uint32_t>(N));
+                    }
+                    // LCOV_EXCL_START
+                    else if (N <= 0xffffffffffffffff)
+                    {
+                        oa->write_character(0x7b);
+                        write_number(static_cast<uint64_t>(N));
+                    }
+                    // LCOV_EXCL_STOP
+
+                    // step 2: write the string
+                    oa->write_characters(reinterpret_cast<const uint8_t*>(j.m_value.string->c_str()),
+                                         j.m_value.string->size());
+                    break;
+                }
+
+                case value_t::array:
+                {
+                    // step 1: write control byte and the array size
+                    const auto N = j.m_value.array->size();
+                    if (N <= 0x17)
+                    {
+                        write_number(static_cast<uint8_t>(0x80 + N));
+                    }
+                    else if (N <= 0xff)
+                    {
+                        oa->write_character(0x98);
+                        write_number(static_cast<uint8_t>(N));
+                    }
+                    else if (N <= 0xffff)
+                    {
+                        oa->write_character(0x99);
+                        write_number(static_cast<uint16_t>(N));
+                    }
+                    else if (N <= 0xffffffff)
+                    {
+                        oa->write_character(0x9a);
+                        write_number(static_cast<uint32_t>(N));
+                    }
+                    // LCOV_EXCL_START
+                    else if (N <= 0xffffffffffffffff)
+                    {
+                        oa->write_character(0x9b);
+                        write_number(static_cast<uint64_t>(N));
+                    }
+                    // LCOV_EXCL_STOP
+
+                    // step 2: write each element
+                    for (const auto& el : *j.m_value.array)
+                    {
+                        write_cbor(el);
+                    }
+                    break;
+                }
+
+                case value_t::object:
+                {
+                    // step 1: write control byte and the object size
+                    const auto N = j.m_value.object->size();
+                    if (N <= 0x17)
+                    {
+                        write_number(static_cast<uint8_t>(0xa0 + N));
+                    }
+                    else if (N <= 0xff)
+                    {
+                        oa->write_character(0xb8);
+                        write_number(static_cast<uint8_t>(N));
+                    }
+                    else if (N <= 0xffff)
+                    {
+                        oa->write_character(0xb9);
+                        write_number(static_cast<uint16_t>(N));
+                    }
+                    else if (N <= 0xffffffff)
+                    {
+                        oa->write_character(0xba);
+                        write_number(static_cast<uint32_t>(N));
+                    }
+                    // LCOV_EXCL_START
+                    else if (N <= 0xffffffffffffffff)
+                    {
+                        oa->write_character(0xbb);
+                        write_number(static_cast<uint64_t>(N));
+                    }
+                    // LCOV_EXCL_STOP
+
+                    // step 2: write each element
+                    for (const auto& el : *j.m_value.object)
+                    {
+                        write_cbor(el.first);
+                        write_cbor(el.second);
+                    }
+                    break;
+                }
+
+                default:
+                {
+                    break;
+                }
+            }
+        }
+
+        /*!
+        @brief[in] j  JSON value to serialize
+        */
+        void write_msgpack(const basic_json& j)
+        {
+            switch (j.type())
+            {
+                case value_t::null:
+                {
+                    // nil
+                    oa->write_character(0xc0);
+                    break;
+                }
+
+                case value_t::boolean:
+                {
+                    // true and false
+                    oa->write_character(j.m_value.boolean ? 0xc3 : 0xc2);
+                    break;
+                }
+
+                case value_t::number_integer:
+                {
+                    if (j.m_value.number_integer >= 0)
+                    {
+                        // MessagePack does not differentiate between positive
+                        // signed integers and unsigned integers. Therefore, we
+                        // used the code from the value_t::number_unsigned case
+                        // here.
+                        if (j.m_value.number_unsigned < 128)
+                        {
+                            // positive fixnum
+                            write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                        {
+                            // uint 8
+                            oa->write_character(0xcc);
+                            write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                        {
+                            // uint 16
+                            oa->write_character(0xcd);
+                            write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                        {
+                            // uint 32
+                            oa->write_character(0xce);
+                            write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                        {
+                            // uint 64
+                            oa->write_character(0xcf);
+                            write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                        }
+                    }
+                    else
+                    {
+                        if (j.m_value.number_integer >= -32)
+                        {
+                            // negative fixnum
+                            write_number(static_cast<int8_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
+                        {
+                            // int 8
+                            oa->write_character(0xd0);
+                            write_number(static_cast<int8_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
+                        {
+                            // int 16
+                            oa->write_character(0xd1);
+                            write_number(static_cast<int16_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
+                        {
+                            // int 32
+                            oa->write_character(0xd2);
+                            write_number(static_cast<int32_t>(j.m_value.number_integer));
+                        }
+                        else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
+                        {
+                            // int 64
+                            oa->write_character(0xd3);
+                            write_number(static_cast<int64_t>(j.m_value.number_integer));
+                        }
+                    }
+                    break;
+                }
+
+                case value_t::number_unsigned:
+                {
+                    if (j.m_value.number_unsigned < 128)
+                    {
+                        // positive fixnum
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        // uint 8
+                        oa->write_character(0xcc);
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        // uint 16
+                        oa->write_character(0xcd);
+                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        // uint 32
+                        oa->write_character(0xce);
+                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                    {
+                        // uint 64
+                        oa->write_character(0xcf);
+                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    }
+                    break;
+                }
+
+                case value_t::number_float:
+                {
+                    // float 64
+                    oa->write_character(0xcb);
+                    write_number(j.m_value.number_float);
+                    break;
+                }
+
+                case value_t::string:
+                {
+                    // step 1: write control byte and the string length
+                    const auto N = j.m_value.string->size();
+                    if (N <= 31)
+                    {
+                        // fixstr
+                        write_number(static_cast<uint8_t>(0xa0 | N));
+                    }
+                    else if (N <= 255)
+                    {
+                        // str 8
+                        oa->write_character(0xd9);
+                        write_number(static_cast<uint8_t>(N));
+                    }
+                    else if (N <= 65535)
+                    {
+                        // str 16
+                        oa->write_character(0xda);
+                        write_number(static_cast<uint16_t>(N));
+                    }
+                    else if (N <= 4294967295)
+                    {
+                        // str 32
+                        oa->write_character(0xdb);
+                        write_number(static_cast<uint32_t>(N));
+                    }
+
+                    // step 2: write the string
+                    oa->write_characters(reinterpret_cast<const uint8_t*>(j.m_value.string->c_str()),
+                                         j.m_value.string->size());
+                    break;
+                }
+
+                case value_t::array:
+                {
+                    // step 1: write control byte and the array size
+                    const auto N = j.m_value.array->size();
+                    if (N <= 15)
+                    {
+                        // fixarray
+                        write_number(static_cast<uint8_t>(0x90 | N));
+                    }
+                    else if (N <= 0xffff)
+                    {
+                        // array 16
+                        oa->write_character(0xdc);
+                        write_number(static_cast<uint16_t>(N));
+                    }
+                    else if (N <= 0xffffffff)
+                    {
+                        // array 32
+                        oa->write_character(0xdd);
+                        write_number(static_cast<uint32_t>(N));
+                    }
+
+                    // step 2: write each element
+                    for (const auto& el : *j.m_value.array)
+                    {
+                        write_msgpack(el);
+                    }
+                    break;
+                }
+
+                case value_t::object:
+                {
+                    // step 1: write control byte and the object size
+                    const auto N = j.m_value.object->size();
+                    if (N <= 15)
+                    {
+                        // fixmap
+                        write_number(static_cast<uint8_t>(0x80 | (N & 0xf)));
+                    }
+                    else if (N <= 65535)
+                    {
+                        // map 16
+                        oa->write_character(0xde);
+                        write_number(static_cast<uint16_t>(N));
+                    }
+                    else if (N <= 4294967295)
+                    {
+                        // map 32
+                        oa->write_character(0xdf);
+                        write_number(static_cast<uint32_t>(N));
+                    }
+
+                    // step 2: write each element
+                    for (const auto& el : *j.m_value.object)
+                    {
+                        write_msgpack(el.first);
+                        write_msgpack(el.second);
+                    }
+                    break;
+                }
+
+                default:
+                {
+                    break;
+                }
+            }
+        }
+
+      private:
+        /*
+        @brief write a number to output input
+
+        @param[in] n number of type @a T
+        @tparam T the type of the number
+
+        @note This function needs to respect the system's endianess, because
+              bytes in CBOR and MessagePack are stored in network order (big
+              endian) and therefore need reordering on little endian systems.
+        */
+        template<typename T>
+        void write_number(T n)
+        {
+            // step 1: write number to array of length T
+            std::array<uint8_t, sizeof(T)> vec;
+            std::memcpy(vec.data(), &n, sizeof(T));
+
+            // step 2: write array to output (with possible reordering)
+            for (size_t i = 0; i < sizeof(T); ++i)
+            {
+                // reverse byte order prior to conversion if necessary
+                if (is_little_endian)
+                {
+                    oa->write_character(vec[sizeof(T) - i - 1]);
+                }
+                else
+                {
+                    oa->write_character(vec[i]);  // LCOV_EXCL_LINE
+                }
+            }
+        }
+
+      private:
+        /// whether we can assume little endianess
+        const bool is_little_endian = true;
+
+        /// the output
+        output_adapter_t<uint8_t> oa = nullptr;
+    };
+
+  public:
+    /*!
+    @brief create a CBOR serialization of a given JSON value
+
+    Serializes a given JSON value @a j to a byte vector using the CBOR (Concise
+    Binary Object Representation) serialization format. CBOR is a binary
+    serialization format which aims to be more compact than JSON itself, yet
+    more efficient to parse.
+
+    The library uses the following mapping from JSON values types to
+    CBOR types according to the CBOR specification (RFC 7049):
+
+    JSON value type | value/range                                | CBOR type                          | first byte
+    --------------- | ------------------------------------------ | ---------------------------------- | ---------------
+    null            | `null`                                     | Null                               | 0xf6
+    boolean         | `true`                                     | True                               | 0xf5
+    boolean         | `false`                                    | False                              | 0xf4
+    number_integer  | -9223372036854775808..-2147483649          | Negative integer (8 bytes follow)  | 0x3b
+    number_integer  | -2147483648..-32769                        | Negative integer (4 bytes follow)  | 0x3a
+    number_integer  | -32768..-129                               | Negative integer (2 bytes follow)  | 0x39
+    number_integer  | -128..-25                                  | Negative integer (1 byte follow)   | 0x38
+    number_integer  | -24..-1                                    | Negative integer                   | 0x20..0x37
+    number_integer  | 0..23                                      | Integer                            | 0x00..0x17
+    number_integer  | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
+    number_integer  | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
+    number_integer  | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1a
+    number_integer  | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1b
+    number_unsigned | 0..23                                      | Integer                            | 0x00..0x17
+    number_unsigned | 24..255                                    | Unsigned integer (1 byte follow)   | 0x18
+    number_unsigned | 256..65535                                 | Unsigned integer (2 bytes follow)  | 0x19
+    number_unsigned | 65536..4294967295                          | Unsigned integer (4 bytes follow)  | 0x1a
+    number_unsigned | 4294967296..18446744073709551615           | Unsigned integer (8 bytes follow)  | 0x1b
+    number_float    | *any value*                                | Double-Precision Float             | 0xfb
+    string          | *length*: 0..23                            | UTF-8 string                       | 0x60..0x77
+    string          | *length*: 23..255                          | UTF-8 string (1 byte follow)       | 0x78
+    string          | *length*: 256..65535                       | UTF-8 string (2 bytes follow)      | 0x79
+    string          | *length*: 65536..4294967295                | UTF-8 string (4 bytes follow)      | 0x7a
+    string          | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow)      | 0x7b
+    array           | *size*: 0..23                              | array                              | 0x80..0x97
+    array           | *size*: 23..255                            | array (1 byte follow)              | 0x98
+    array           | *size*: 256..65535                         | array (2 bytes follow)             | 0x99
+    array           | *size*: 65536..4294967295                  | array (4 bytes follow)             | 0x9a
+    array           | *size*: 4294967296..18446744073709551615   | array (8 bytes follow)             | 0x9b
+    object          | *size*: 0..23                              | map                                | 0xa0..0xb7
+    object          | *size*: 23..255                            | map (1 byte follow)                | 0xb8
+    object          | *size*: 256..65535                         | map (2 bytes follow)               | 0xb9
+    object          | *size*: 65536..4294967295                  | map (4 bytes follow)               | 0xba
+    object          | *size*: 4294967296..18446744073709551615   | map (8 bytes follow)               | 0xbb
+
+    @note The mapping is **complete** in the sense that any JSON value type
+          can be converted to a CBOR value.
+
+    @note The following CBOR types are not used in the conversion:
+          - byte strings (0x40..0x5f)
+          - UTF-8 strings terminated by "break" (0x7f)
+          - arrays terminated by "break" (0x9f)
+          - maps terminated by "break" (0xbf)
+          - date/time (0xc0..0xc1)
+          - bignum (0xc2..0xc3)
+          - decimal fraction (0xc4)
+          - bigfloat (0xc5)
+          - tagged items (0xc6..0xd4, 0xd8..0xdb)
+          - expected conversions (0xd5..0xd7)
+          - simple values (0xe0..0xf3, 0xf8)
+          - undefined (0xf7)
+          - half and single-precision floats (0xf9-0xfa)
+          - break (0xff)
+
+    @param[in] j  JSON value to serialize
+    @return MessagePack serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in CBOR format.,to_cbor}
+
+    @sa http://cbor.io
+    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
+        analogous deserialization
+    @sa @ref to_msgpack(const basic_json& for the related MessagePack format
+
+    @since version 2.0.9
+    */
+    static std::vector<uint8_t> to_cbor(const basic_json& j)
+    {
+        std::vector<uint8_t> result;
+        binary_writer bw(output_adapter<uint8_t>::create(result));
+        bw.write_cbor(j);
+        return result;
+    }
+
+    /*!
+    @brief create a MessagePack serialization of a given JSON value
+
+    Serializes a given JSON value @a j to a byte vector using the MessagePack
+    serialization format. MessagePack is a binary serialization format which
+    aims to be more compact than JSON itself, yet more efficient to parse.
+
+    The library uses the following mapping from JSON values types to
+    MessagePack types according to the MessagePack specification:
+
+    JSON value type | value/range                       | MessagePack type | first byte
+    --------------- | --------------------------------- | ---------------- | ----------
+    null            | `null`                            | nil              | 0xc0
+    boolean         | `true`                            | true             | 0xc3
+    boolean         | `false`                           | false            | 0xc2
+    number_integer  | -9223372036854775808..-2147483649 | int64            | 0xd3
+    number_integer  | -2147483648..-32769               | int32            | 0xd2
+    number_integer  | -32768..-129                      | int16            | 0xd1
+    number_integer  | -128..-33                         | int8             | 0xd0
+    number_integer  | -32..-1                           | negative fixint  | 0xe0..0xff
+    number_integer  | 0..127                            | positive fixint  | 0x00..0x7f
+    number_integer  | 128..255                          | uint 8           | 0xcc
+    number_integer  | 256..65535                        | uint 16          | 0xcd
+    number_integer  | 65536..4294967295                 | uint 32          | 0xce
+    number_integer  | 4294967296..18446744073709551615  | uint 64          | 0xcf
+    number_unsigned | 0..127                            | positive fixint  | 0x00..0x7f
+    number_unsigned | 128..255                          | uint 8           | 0xcc
+    number_unsigned | 256..65535                        | uint 16          | 0xcd
+    number_unsigned | 65536..4294967295                 | uint 32          | 0xce
+    number_unsigned | 4294967296..18446744073709551615  | uint 64          | 0xcf
+    number_float    | *any value*                       | float 64         | 0xcb
+    string          | *length*: 0..31                   | fixstr           | 0xa0..0xbf
+    string          | *length*: 32..255                 | str 8            | 0xd9
+    string          | *length*: 256..65535              | str 16           | 0xda
+    string          | *length*: 65536..4294967295       | str 32           | 0xdb
+    array           | *size*: 0..15                     | fixarray         | 0x90..0x9f
+    array           | *size*: 16..65535                 | array 16         | 0xdc
+    array           | *size*: 65536..4294967295         | array 32         | 0xdd
+    object          | *size*: 0..15                     | fix map          | 0x80..0x8f
+    object          | *size*: 16..65535                 | map 16           | 0xde
+    object          | *size*: 65536..4294967295         | map 32           | 0xdf
+
+    @note The mapping is **complete** in the sense that any JSON value type
+          can be converted to a MessagePack value.
+
+    @note The following values can **not** be converted to a MessagePack value:
+          - strings with more than 4294967295 bytes
+          - arrays with more than 4294967295 elements
+          - objects with more than 4294967295 elements
+
+    @note The following MessagePack types are not used in the conversion:
+          - bin 8 - bin 32 (0xc4..0xc6)
+          - ext 8 - ext 32 (0xc7..0xc9)
+          - float 32 (0xca)
+          - fixext 1 - fixext 16 (0xd4..0xd8)
+
+    @note Any MessagePack output created @ref to_msgpack can be successfully
+          parsed by @ref from_msgpack.
+
+    @param[in] j  JSON value to serialize
+    @return MessagePack serialization as byte vector
+
+    @complexity Linear in the size of the JSON value @a j.
+
+    @liveexample{The example shows the serialization of a JSON value to a byte
+    vector in MessagePack format.,to_msgpack}
+
+    @sa http://msgpack.org
+    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
+        analogous deserialization
+    @sa @ref to_cbor(const basic_json& for the related CBOR format
+
+    @since version 2.0.9
+    */
+    static std::vector<uint8_t> to_msgpack(const basic_json& j)
+    {
+        std::vector<uint8_t> result;
+        binary_writer bw(output_adapter<uint8_t>::create(result));
+        bw.write_msgpack(j);
+        return result;
+    }
+
+    /*!
+    @brief create a JSON value from a byte vector in CBOR format
+
+    Deserializes a given byte vector @a v to a JSON value using the CBOR
+    (Concise Binary Object Representation) serialization format.
+
+    The library maps CBOR types to JSON value types as follows:
+
+    CBOR type              | JSON value type | first byte
+    ---------------------- | --------------- | ----------
+    Integer                | number_unsigned | 0x00..0x17
+    Unsigned integer       | number_unsigned | 0x18
+    Unsigned integer       | number_unsigned | 0x19
+    Unsigned integer       | number_unsigned | 0x1a
+    Unsigned integer       | number_unsigned | 0x1b
+    Negative integer       | number_integer  | 0x20..0x37
+    Negative integer       | number_integer  | 0x38
+    Negative integer       | number_integer  | 0x39
+    Negative integer       | number_integer  | 0x3a
+    Negative integer       | number_integer  | 0x3b
+    Negative integer       | number_integer  | 0x40..0x57
+    UTF-8 string           | string          | 0x60..0x77
+    UTF-8 string           | string          | 0x78
+    UTF-8 string           | string          | 0x79
+    UTF-8 string           | string          | 0x7a
+    UTF-8 string           | string          | 0x7b
+    UTF-8 string           | string          | 0x7f
+    array                  | array           | 0x80..0x97
+    array                  | array           | 0x98
+    array                  | array           | 0x99
+    array                  | array           | 0x9a
+    array                  | array           | 0x9b
+    array                  | array           | 0x9f
+    map                    | object          | 0xa0..0xb7
+    map                    | object          | 0xb8
+    map                    | object          | 0xb9
+    map                    | object          | 0xba
+    map                    | object          | 0xbb
+    map                    | object          | 0xbf
+    False                  | `false`         | 0xf4
+    True                   | `true`          | 0xf5
+    Nill                   | `null`          | 0xf6
+    Half-Precision Float   | number_float    | 0xf9
+    Single-Precision Float | number_float    | 0xfa
+    Double-Precision Float | number_float    | 0xfb
+
+    @warning The mapping is **incomplete** in the sense that not all CBOR
+             types can be converted to a JSON value. The following CBOR types
+             are not supported and will yield parse errors (parse_error.112):
+             - byte strings (0x40..0x5f)
+             - date/time (0xc0..0xc1)
+             - bignum (0xc2..0xc3)
+             - decimal fraction (0xc4)
+             - bigfloat (0xc5)
+             - tagged items (0xc6..0xd4, 0xd8..0xdb)
+             - expected conversions (0xd5..0xd7)
+             - simple values (0xe0..0xf3, 0xf8)
+             - undefined (0xf7)
+
+    @warning CBOR allows map keys of any type, whereas JSON only allows
+             strings as keys in object values. Therefore, CBOR maps with keys
+             other than UTF-8 strings are rejected (parse_error.113).
+
+    @note Any CBOR output created @ref to_cbor can be successfully parsed by
+          @ref from_cbor.
+
+    @param[in] v  a byte vector in CBOR format
+    @param[in] start_index the index to start reading from @a v (0 by default)
+    @return deserialized JSON value
+
+    @throw parse_error.110 if the given vector ends prematurely
+    @throw parse_error.112 if unsupported features from CBOR were
+    used in the given vector @a v or if the input is not valid CBOR
+    @throw parse_error.113 if a string was expected as map key, but not found
+
+    @complexity Linear in the size of the byte vector @a v.
+
+    @liveexample{The example shows the deserialization of a byte vector in CBOR
+    format to a JSON value.,from_cbor}
+
+    @sa http://cbor.io
+    @sa @ref to_cbor(const basic_json&) for the analogous serialization
+    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
+        related MessagePack format
+
+    @since version 2.0.9, parameter @a start_index since 2.1.1
+    */
+    static basic_json from_cbor(const std::vector<uint8_t>& v,
+                                const size_t start_index = 0)
+    {
+        binary_reader br(input_adapter::create(v.begin() + static_cast<difference_type>(start_index), v.end()));
+        return br.parse_cbor();
+    }
+
+
+    /*!
+    @brief create a JSON value from a byte vector in MessagePack format
+
+    Deserializes a given byte vector @a v to a JSON value using the MessagePack
+    serialization format.
+
+    The library maps MessagePack types to JSON value types as follows:
+
+    MessagePack type | JSON value type | first byte
+    ---------------- | --------------- | ----------
+    positive fixint  | number_unsigned | 0x00..0x7f
+    fixmap           | object          | 0x80..0x8f
+    fixarray         | array           | 0x90..0x9f
+    fixstr           | string          | 0xa0..0xbf
+    nil              | `null`          | 0xc0
+    false            | `false`         | 0xc2
+    true             | `true`          | 0xc3
+    float 32         | number_float    | 0xca
+    float 64         | number_float    | 0xcb
+    uint 8           | number_unsigned | 0xcc
+    uint 16          | number_unsigned | 0xcd
+    uint 32          | number_unsigned | 0xce
+    uint 64          | number_unsigned | 0xcf
+    int 8            | number_integer  | 0xd0
+    int 16           | number_integer  | 0xd1
+    int 32           | number_integer  | 0xd2
+    int 64           | number_integer  | 0xd3
+    str 8            | string          | 0xd9
+    str 16           | string          | 0xda
+    str 32           | string          | 0xdb
+    array 16         | array           | 0xdc
+    array 32         | array           | 0xdd
+    map 16           | object          | 0xde
+    map 32           | object          | 0xdf
+    negative fixint  | number_integer  | 0xe0-0xff
+
+    @warning The mapping is **incomplete** in the sense that not all
+             MessagePack types can be converted to a JSON value. The following
+             MessagePack types are not supported and will yield parse errors:
+              - bin 8 - bin 32 (0xc4..0xc6)
+              - ext 8 - ext 32 (0xc7..0xc9)
+              - fixext 1 - fixext 16 (0xd4..0xd8)
+
+    @note Any MessagePack output created @ref to_msgpack can be successfully
+          parsed by @ref from_msgpack.
+
+    @param[in] v  a byte vector in MessagePack format
+    @param[in] start_index the index to start reading from @a v (0 by default)
+    @return deserialized JSON value
+
+    @throw parse_error.110 if the given vector ends prematurely
+    @throw parse_error.112 if unsupported features from MessagePack were
+    used in the given vector @a v or if the input is not valid MessagePack
+    @throw parse_error.113 if a string was expected as map key, but not found
+
+    @complexity Linear in the size of the byte vector @a v.
+
+    @liveexample{The example shows the deserialization of a byte vector in
+    MessagePack format to a JSON value.,from_msgpack}
+
+    @sa http://msgpack.org
+    @sa @ref to_msgpack(const basic_json&) for the analogous serialization
+    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
+        related CBOR format
+
+    @since version 2.0.9, parameter @a start_index since 2.1.1
+    */
+    static basic_json from_msgpack(const std::vector<uint8_t>& v,
+                                   const size_t start_index = 0)
+    {
+        binary_reader br(input_adapter::create(v.begin() + static_cast<difference_type>(start_index), v.end()));
+        return br.parse_msgpack();
+    }
+
+    /// @}
+
     //////////////////////
     // lexer and parser //
     //////////////////////
 
+  private:
     /*!
     @brief lexical analysis
 
-    This class organizes the lexical analysis during JSON deserialization. The
-    core of it is a scanner generated by [re2c](http://re2c.org) that
-    processes a buffer and recognizes tokens according to RFC 7159.
+    This class organizes the lexical analysis during JSON deserialization.
     */
     class lexer
     {
@@ -9567,9 +11122,9 @@ class basic_json
             literal_false,   ///< the `false` literal
             literal_null,    ///< the `null` literal
             value_string,    ///< a string -- use get_string() for actual value
-            value_unsigned,  ///< an unsigned integer -- use get_number() for actual value
-            value_integer,   ///< a signed integer -- use get_number() for actual value
-            value_float,     ///< an floating point number -- use get_number() for actual value
+            value_unsigned,  ///< an unsigned integer -- use get_number_unsigned() for actual value
+            value_integer,   ///< a signed integer -- use get_number_integer() for actual value
+            value_float,     ///< an floating point number -- use get_number_float() for actual value
             begin_array,     ///< the character for array begin `[`
             begin_object,    ///< the character for object begin `{`
             end_array,       ///< the character for array end `]`
@@ -9577,137 +11132,12 @@ class basic_json
             name_separator,  ///< the name separator `:`
             value_separator, ///< the value separator `,`
             parse_error,     ///< indicating a parse error
-            end_of_input     ///< indicating the end of the input buffer
+            end_of_input,    ///< indicating the end of the input buffer
+            literal_or_value ///< a literal or the begin of a value (only for diagnostics)
         };
 
-        /// the char type to use in the lexer
-        using lexer_char_t = unsigned char;
-
-        /// a lexer from a buffer with given length
-        lexer(const lexer_char_t* buff, const size_t len) noexcept
-            : m_content(buff)
-        {
-            assert(m_content != nullptr);
-            m_start = m_cursor = m_content;
-            m_limit = m_content + len;
-        }
-
-        /// a lexer from an input stream
-        explicit lexer(std::istream& s)
-            : m_stream(&s), m_line_buffer()
-        {
-            // immediately abort if stream is erroneous
-            if (s.fail())
-            {
-                JSON_THROW(std::invalid_argument("stream error"));
-            }
-
-            // fill buffer
-            fill_line_buffer();
-
-            // skip UTF-8 byte-order mark
-            if (m_line_buffer.size() >= 3 and m_line_buffer.substr(0, 3) == "\xEF\xBB\xBF")
-            {
-                m_line_buffer[0] = ' ';
-                m_line_buffer[1] = ' ';
-                m_line_buffer[2] = ' ';
-            }
-        }
-
-        // switch off unwanted functions (due to pointer members)
-        lexer() = delete;
-        lexer(const lexer&) = delete;
-        lexer operator=(const lexer&) = delete;
-
-        /*!
-        @brief create a string from one or two Unicode code points
-
-        There are two cases: (1) @a codepoint1 is in the Basic Multilingual
-        Plane (U+0000 through U+FFFF) and @a codepoint2 is 0, or (2)
-        @a codepoint1 and @a codepoint2 are a UTF-16 surrogate pair to
-        represent a code point above U+FFFF.
-
-        @param[in] codepoint1  the code point (can be high surrogate)
-        @param[in] codepoint2  the code point (can be low surrogate or 0)
-
-        @return string representation of the code point; the length of the
-        result string is between 1 and 4 characters.
-
-        @throw std::out_of_range if code point is > 0x10ffff; example: `"code
-        points above 0x10FFFF are invalid"`
-        @throw std::invalid_argument if the low surrogate is invalid; example:
-        `""missing or wrong low surrogate""`
-
-        @complexity Constant.
-
-        @see <http://en.wikipedia.org/wiki/UTF-8#Sample_code>
-        */
-        static string_t to_unicode(const std::size_t codepoint1,
-                                   const std::size_t codepoint2 = 0)
-        {
-            // calculate the code point from the given code points
-            std::size_t codepoint = codepoint1;
-
-            // check if codepoint1 is a high surrogate
-            if (codepoint1 >= 0xD800 and codepoint1 <= 0xDBFF)
-            {
-                // check if codepoint2 is a low surrogate
-                if (codepoint2 >= 0xDC00 and codepoint2 <= 0xDFFF)
-                {
-                    codepoint =
-                        // high surrogate occupies the most significant 22 bits
-                        (codepoint1 << 10)
-                        // low surrogate occupies the least significant 15 bits
-                        + codepoint2
-                        // there is still the 0xD800, 0xDC00 and 0x10000 noise
-                        // in the result so we have to subtract with:
-                        // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
-                        - 0x35FDC00;
-                }
-                else
-                {
-                    JSON_THROW(std::invalid_argument("missing or wrong low surrogate"));
-                }
-            }
-
-            string_t result;
-
-            if (codepoint < 0x80)
-            {
-                // 1-byte characters: 0xxxxxxx (ASCII)
-                result.append(1, static_cast<typename string_t::value_type>(codepoint));
-            }
-            else if (codepoint <= 0x7ff)
-            {
-                // 2-byte characters: 110xxxxx 10xxxxxx
-                result.append(1, static_cast<typename string_t::value_type>(0xC0 | ((codepoint >> 6) & 0x1F)));
-                result.append(1, static_cast<typename string_t::value_type>(0x80 | (codepoint & 0x3F)));
-            }
-            else if (codepoint <= 0xffff)
-            {
-                // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
-                result.append(1, static_cast<typename string_t::value_type>(0xE0 | ((codepoint >> 12) & 0x0F)));
-                result.append(1, static_cast<typename string_t::value_type>(0x80 | ((codepoint >> 6) & 0x3F)));
-                result.append(1, static_cast<typename string_t::value_type>(0x80 | (codepoint & 0x3F)));
-            }
-            else if (codepoint <= 0x10ffff)
-            {
-                // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                result.append(1, static_cast<typename string_t::value_type>(0xF0 | ((codepoint >> 18) & 0x07)));
-                result.append(1, static_cast<typename string_t::value_type>(0x80 | ((codepoint >> 12) & 0x3F)));
-                result.append(1, static_cast<typename string_t::value_type>(0x80 | ((codepoint >> 6) & 0x3F)));
-                result.append(1, static_cast<typename string_t::value_type>(0x80 | (codepoint & 0x3F)));
-            }
-            else
-            {
-                JSON_THROW(std::out_of_range("code points above 0x10FFFF are invalid"));
-            }
-
-            return result;
-        }
-
         /// return name of values of type token_type (only used for errors)
-        static std::string token_type_name(const token_type t)
+        static const char* token_type_name(const token_type t) noexcept
         {
             switch (t)
             {
@@ -9741,6 +11171,8 @@ class basic_json
                     return "<parse error>";
                 case token_type::end_of_input:
                     return "end of input";
+                case token_type::literal_or_value:
+                    return "'[', '{', or a literal";
                 default:
                 {
                     // catch non-enum values
@@ -9749,1600 +11181,1438 @@ class basic_json
             }
         }
 
-        /*!
-        This function implements a scanner for JSON. It is specified using
-        regular expressions that try to follow RFC 7159 as close as possible.
-        These regular expressions are then translated into a minimized
-        deterministic finite automaton (DFA) by the tool
-        [re2c](http://re2c.org). As a result, the translated code for this
-        function consists of a large block of code with `goto` jumps.
+        explicit lexer(input_adapter_t adapter)
+            : ia(adapter), decimal_point_char(get_decimal_point())
+        {}
 
-        @return the class of the next token read from the buffer
+        // delete because of pointer members
+        lexer(const lexer&) = delete;
+        lexer& operator=(lexer&) = delete;
 
-        @complexity Linear in the length of the input.\n
+      private:
+        /////////////////////
+        // locales
+        /////////////////////
 
-        Proposition: The loop below will always terminate for finite input.\n
-
-        Proof (by contradiction): Assume a finite input. To loop forever, the
-        loop must never hit code with a `break` statement. The only code
-        snippets without a `break` statement are the continue statements for
-        whitespace and byte-order-marks. To loop forever, the input must be an
-        infinite sequence of whitespace or byte-order-marks. This contradicts
-        the assumption of finite input, q.e.d.
-        */
-        token_type scan()
+        /// return the locale-dependent decimal point
+        static char get_decimal_point() noexcept
         {
+            const auto loc = localeconv();
+            assert(loc != nullptr);
+            return (loc->decimal_point == nullptr) ? '.' : loc->decimal_point[0];
+        }
+
+        /////////////////////
+        // scan functions
+        /////////////////////
+
+        /*!
+        @brief get codepoint from 4 hex characters following `\u`
+
+        @return codepoint or -1 in case of an error (e.g. EOF or non-hex
+                character)
+        */
+        int get_codepoint()
+        {
+            // this function only makes sense after reading `\u`
+            assert(current == 'u');
+            int codepoint = 0;
+
+            // byte 1: \uXxxx
+            switch (get())
+            {
+                case '0':
+                    break;
+                case '1':
+                    codepoint += 0x1000;
+                    break;
+                case '2':
+                    codepoint += 0x2000;
+                    break;
+                case '3':
+                    codepoint += 0x3000;
+                    break;
+                case '4':
+                    codepoint += 0x4000;
+                    break;
+                case '5':
+                    codepoint += 0x5000;
+                    break;
+                case '6':
+                    codepoint += 0x6000;
+                    break;
+                case '7':
+                    codepoint += 0x7000;
+                    break;
+                case '8':
+                    codepoint += 0x8000;
+                    break;
+                case '9':
+                    codepoint += 0x9000;
+                    break;
+                case 'A':
+                case 'a':
+                    codepoint += 0xa000;
+                    break;
+                case 'B':
+                case 'b':
+                    codepoint += 0xb000;
+                    break;
+                case 'C':
+                case 'c':
+                    codepoint += 0xc000;
+                    break;
+                case 'D':
+                case 'd':
+                    codepoint += 0xd000;
+                    break;
+                case 'E':
+                case 'e':
+                    codepoint += 0xe000;
+                    break;
+                case 'F':
+                case 'f':
+                    codepoint += 0xf000;
+                    break;
+                default:
+                    return -1;
+            }
+
+            // byte 2: \uxXxx
+            switch (get())
+            {
+                case '0':
+                    break;
+                case '1':
+                    codepoint += 0x0100;
+                    break;
+                case '2':
+                    codepoint += 0x0200;
+                    break;
+                case '3':
+                    codepoint += 0x0300;
+                    break;
+                case '4':
+                    codepoint += 0x0400;
+                    break;
+                case '5':
+                    codepoint += 0x0500;
+                    break;
+                case '6':
+                    codepoint += 0x0600;
+                    break;
+                case '7':
+                    codepoint += 0x0700;
+                    break;
+                case '8':
+                    codepoint += 0x0800;
+                    break;
+                case '9':
+                    codepoint += 0x0900;
+                    break;
+                case 'A':
+                case 'a':
+                    codepoint += 0x0a00;
+                    break;
+                case 'B':
+                case 'b':
+                    codepoint += 0x0b00;
+                    break;
+                case 'C':
+                case 'c':
+                    codepoint += 0x0c00;
+                    break;
+                case 'D':
+                case 'd':
+                    codepoint += 0x0d00;
+                    break;
+                case 'E':
+                case 'e':
+                    codepoint += 0x0e00;
+                    break;
+                case 'F':
+                case 'f':
+                    codepoint += 0x0f00;
+                    break;
+                default:
+                    return -1;
+            }
+
+            // byte 3: \uxxXx
+            switch (get())
+            {
+                case '0':
+                    break;
+                case '1':
+                    codepoint += 0x0010;
+                    break;
+                case '2':
+                    codepoint += 0x0020;
+                    break;
+                case '3':
+                    codepoint += 0x0030;
+                    break;
+                case '4':
+                    codepoint += 0x0040;
+                    break;
+                case '5':
+                    codepoint += 0x0050;
+                    break;
+                case '6':
+                    codepoint += 0x0060;
+                    break;
+                case '7':
+                    codepoint += 0x0070;
+                    break;
+                case '8':
+                    codepoint += 0x0080;
+                    break;
+                case '9':
+                    codepoint += 0x0090;
+                    break;
+                case 'A':
+                case 'a':
+                    codepoint += 0x00a0;
+                    break;
+                case 'B':
+                case 'b':
+                    codepoint += 0x00b0;
+                    break;
+                case 'C':
+                case 'c':
+                    codepoint += 0x00c0;
+                    break;
+                case 'D':
+                case 'd':
+                    codepoint += 0x00d0;
+                    break;
+                case 'E':
+                case 'e':
+                    codepoint += 0x00e0;
+                    break;
+                case 'F':
+                case 'f':
+                    codepoint += 0x00f0;
+                    break;
+                default:
+                    return -1;
+            }
+
+            // byte 4: \uxxxX
+            switch (get())
+            {
+                case '0':
+                    break;
+                case '1':
+                    codepoint += 0x0001;
+                    break;
+                case '2':
+                    codepoint += 0x0002;
+                    break;
+                case '3':
+                    codepoint += 0x0003;
+                    break;
+                case '4':
+                    codepoint += 0x0004;
+                    break;
+                case '5':
+                    codepoint += 0x0005;
+                    break;
+                case '6':
+                    codepoint += 0x0006;
+                    break;
+                case '7':
+                    codepoint += 0x0007;
+                    break;
+                case '8':
+                    codepoint += 0x0008;
+                    break;
+                case '9':
+                    codepoint += 0x0009;
+                    break;
+                case 'A':
+                case 'a':
+                    codepoint += 0x000a;
+                    break;
+                case 'B':
+                case 'b':
+                    codepoint += 0x000b;
+                    break;
+                case 'C':
+                case 'c':
+                    codepoint += 0x000c;
+                    break;
+                case 'D':
+                case 'd':
+                    codepoint += 0x000d;
+                    break;
+                case 'E':
+                case 'e':
+                    codepoint += 0x000e;
+                    break;
+                case 'F':
+                case 'f':
+                    codepoint += 0x000f;
+                    break;
+                default:
+                    return -1;
+            }
+
+            return codepoint;
+        }
+
+        /*!
+        @brief scan a string literal
+
+        This function scans a string according to Sect. 7 of RFC 7159. While
+        scanning, bytes are escaped and copied into buffer yytext. Then the
+        function returns successfully, yytext is null-terminated and yylen
+        contains the number of bytes in the string.
+
+        @return token_type::value_string if string could be successfully
+                scanned, token_type::parse_error otherwise
+
+        @note In case of errors, variable error_message contains a textual
+              description.
+        */
+        token_type scan_string()
+        {
+            // reset yytext (ignore opening quote)
+            reset();
+
+            // we entered the function by reading an open quote
+            assert(current == '\"');
+
             while (true)
             {
-                // pointer for backtracking information
-                m_marker = nullptr;
-
-                // remember the begin of the token
-                m_start = m_cursor;
-                assert(m_start != nullptr);
-
-
+                // get next character
+                switch (get())
                 {
-                    lexer_char_t yych;
-                    unsigned int yyaccept = 0;
-                    static const unsigned char yybm[] =
+                    // end of file while parsing string
+                    case std::char_traits<char>::eof():
                     {
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,  32,  32,   0,   0,  32,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        160, 128,   0, 128, 128, 128, 128, 128,
-                        128, 128, 128, 128, 128, 128, 128, 128,
-                        192, 192, 192, 192, 192, 192, 192, 192,
-                        192, 192, 128, 128, 128, 128, 128, 128,
-                        128, 128, 128, 128, 128, 128, 128, 128,
-                        128, 128, 128, 128, 128, 128, 128, 128,
-                        128, 128, 128, 128, 128, 128, 128, 128,
-                        128, 128, 128, 128,   0, 128, 128, 128,
-                        128, 128, 128, 128, 128, 128, 128, 128,
-                        128, 128, 128, 128, 128, 128, 128, 128,
-                        128, 128, 128, 128, 128, 128, 128, 128,
-                        128, 128, 128, 128, 128, 128, 128, 128,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                        0,   0,   0,   0,   0,   0,   0,   0,
-                    };
-                    if ((m_limit - m_cursor) < 5)
-                    {
-                        fill_line_buffer(5);    // LCOV_EXCL_LINE
+                        error_message = "invalid string: missing closing quote";
+                        return token_type::parse_error;
                     }
-                    yych = *m_cursor;
-                    if (yybm[0 + yych] & 32)
+
+                    // closing quote
+                    case '\"':
                     {
-                        goto basic_json_parser_6;
+                        // terminate yytext
+                        add('\0');
+                        --yylen;
+                        return token_type::value_string;
                     }
-                    if (yych <= '[')
+
+                    // escapes
+                    case '\\':
                     {
-                        if (yych <= '-')
+                        switch (get())
                         {
-                            if (yych <= '"')
+                            // quotation mark
+                            case '\"':
+                                add('\"');
+                                break;
+                            // reverse solidus
+                            case '\\':
+                                add('\\');
+                                break;
+                            // solidus
+                            case '/':
+                                add('/');
+                                break;
+                            // backspace
+                            case 'b':
+                                add('\b');
+                                break;
+                            // form feed
+                            case 'f':
+                                add('\f');
+                                break;
+                            // line feed
+                            case 'n':
+                                add('\n');
+                                break;
+                            // carriage return
+                            case 'r':
+                                add('\r');
+                                break;
+                            // tab
+                            case 't':
+                                add('\t');
+                                break;
+
+                            // unicode escapes
+                            case 'u':
                             {
-                                if (yych <= 0x00)
+                                int codepoint;
+                                int codepoint1 = get_codepoint();
+
+                                if (JSON_UNLIKELY(codepoint1 == -1))
                                 {
-                                    goto basic_json_parser_2;
+                                    error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                    return token_type::parse_error;
                                 }
-                                if (yych <= '!')
+
+                                // check if code point is a high surrogate
+                                if (0xD800 <= codepoint1 and codepoint1 <= 0xDBFF)
                                 {
-                                    goto basic_json_parser_4;
+                                    // expect next \uxxxx entry
+                                    if (JSON_LIKELY(get() == '\\' and get() == 'u'))
+                                    {
+                                        const int codepoint2 = get_codepoint();
+
+                                        if (JSON_UNLIKELY(codepoint2 == -1))
+                                        {
+                                            error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                            return token_type::parse_error;
+                                        }
+
+                                        // check if codepoint2 is a low surrogate
+                                        if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF))
+                                        {
+                                            codepoint =
+                                                // high surrogate occupies the most significant 22 bits
+                                                (codepoint1 << 10)
+                                                // low surrogate occupies the least significant 15 bits
+                                                + codepoint2
+                                                // there is still the 0xD800, 0xDC00 and 0x10000 noise
+                                                // in the result so we have to subtract with:
+                                                // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
+                                                - 0x35FDC00;
+                                        }
+                                        else
+                                        {
+                                            error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                            return token_type::parse_error;
+                                        }
+                                    }
+                                    else
+                                    {
+                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                        return token_type::parse_error;
+                                    }
                                 }
-                                goto basic_json_parser_9;
-                            }
-                            else
-                            {
-                                if (yych <= '+')
-                                {
-                                    goto basic_json_parser_4;
-                                }
-                                if (yych <= ',')
-                                {
-                                    goto basic_json_parser_10;
-                                }
-                                goto basic_json_parser_12;
+                                else
+                                {
+                                    if (JSON_UNLIKELY(0xDC00 <= codepoint1 and codepoint1 <= 0xDFFF))
+                                    {
+                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
+                                        return token_type::parse_error;
+                                    }
+
+                                    // only work with first code point
+                                    codepoint = codepoint1;
+                                }
+
+                                // result of the above calculation yields a proper codepoint
+                                assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
+
+                                // translate code point to bytes
+                                if (codepoint < 0x80)
+                                {
+                                    // 1-byte characters: 0xxxxxxx (ASCII)
+                                    add(codepoint);
+                                }
+                                else if (codepoint <= 0x7ff)
+                                {
+                                    // 2-byte characters: 110xxxxx 10xxxxxx
+                                    add(0xC0 | (codepoint >> 6));
+                                    add(0x80 | (codepoint & 0x3F));
+                                }
+                                else if (codepoint <= 0xffff)
+                                {
+                                    // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
+                                    add(0xE0 | (codepoint >> 12));
+                                    add(0x80 | ((codepoint >> 6) & 0x3F));
+                                    add(0x80 | (codepoint & 0x3F));
+                                }
+                                else
+                                {
+                                    // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                                    add(0xF0 | (codepoint >> 18));
+                                    add(0x80 | ((codepoint >> 12) & 0x3F));
+                                    add(0x80 | ((codepoint >> 6) & 0x3F));
+                                    add(0x80 | (codepoint & 0x3F));
+                                }
+
+                                break;
                             }
+
+                            // other characters after escape
+                            default:
+                                error_message = "invalid string: forbidden character after backslash";
+                                return token_type::parse_error;
                         }
-                        else
-                        {
-                            if (yych <= '9')
-                            {
-                                if (yych <= '/')
-                                {
-                                    goto basic_json_parser_4;
-                                }
-                                if (yych <= '0')
-                                {
-                                    goto basic_json_parser_13;
-                                }
-                                goto basic_json_parser_15;
-                            }
-                            else
-                            {
-                                if (yych <= ':')
-                                {
-                                    goto basic_json_parser_17;
-                                }
-                                if (yych <= 'Z')
-                                {
-                                    goto basic_json_parser_4;
-                                }
-                                goto basic_json_parser_19;
-                            }
-                        }
-                    }
-                    else
-                    {
-                        if (yych <= 'n')
-                        {
-                            if (yych <= 'e')
-                            {
-                                if (yych == ']')
-                                {
-                                    goto basic_json_parser_21;
-                                }
-                                goto basic_json_parser_4;
-                            }
-                            else
-                            {
-                                if (yych <= 'f')
-                                {
-                                    goto basic_json_parser_23;
-                                }
-                                if (yych <= 'm')
-                                {
-                                    goto basic_json_parser_4;
-                                }
-                                goto basic_json_parser_24;
-                            }
-                        }
-                        else
-                        {
-                            if (yych <= 'z')
-                            {
-                                if (yych == 't')
-                                {
-                                    goto basic_json_parser_25;
-                                }
-                                goto basic_json_parser_4;
-                            }
-                            else
-                            {
-                                if (yych <= '{')
-                                {
-                                    goto basic_json_parser_26;
-                                }
-                                if (yych == '}')
-                                {
-                                    goto basic_json_parser_28;
-                                }
-                                goto basic_json_parser_4;
-                            }
-                        }
-                    }
-basic_json_parser_2:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::end_of_input;
+
                         break;
                     }
-basic_json_parser_4:
-                    ++m_cursor;
-basic_json_parser_5:
+
+                    // invalid control characters
+                    case 0x00:
+                    case 0x01:
+                    case 0x02:
+                    case 0x03:
+                    case 0x04:
+                    case 0x05:
+                    case 0x06:
+                    case 0x07:
+                    case 0x08:
+                    case 0x09:
+                    case 0x0a:
+                    case 0x0b:
+                    case 0x0c:
+                    case 0x0d:
+                    case 0x0e:
+                    case 0x0f:
+                    case 0x10:
+                    case 0x11:
+                    case 0x12:
+                    case 0x13:
+                    case 0x14:
+                    case 0x15:
+                    case 0x16:
+                    case 0x17:
+                    case 0x18:
+                    case 0x19:
+                    case 0x1a:
+                    case 0x1b:
+                    case 0x1c:
+                    case 0x1d:
+                    case 0x1e:
+                    case 0x1f:
                     {
-                        last_token_type = token_type::parse_error;
+                        error_message = "invalid string: control character must be escaped";
+                        return token_type::parse_error;
+                    }
+
+                    // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))
+                    case 0x20:
+                    case 0x21:
+                    case 0x23:
+                    case 0x24:
+                    case 0x25:
+                    case 0x26:
+                    case 0x27:
+                    case 0x28:
+                    case 0x29:
+                    case 0x2a:
+                    case 0x2b:
+                    case 0x2c:
+                    case 0x2d:
+                    case 0x2e:
+                    case 0x2f:
+                    case 0x30:
+                    case 0x31:
+                    case 0x32:
+                    case 0x33:
+                    case 0x34:
+                    case 0x35:
+                    case 0x36:
+                    case 0x37:
+                    case 0x38:
+                    case 0x39:
+                    case 0x3a:
+                    case 0x3b:
+                    case 0x3c:
+                    case 0x3d:
+                    case 0x3e:
+                    case 0x3f:
+                    case 0x40:
+                    case 0x41:
+                    case 0x42:
+                    case 0x43:
+                    case 0x44:
+                    case 0x45:
+                    case 0x46:
+                    case 0x47:
+                    case 0x48:
+                    case 0x49:
+                    case 0x4a:
+                    case 0x4b:
+                    case 0x4c:
+                    case 0x4d:
+                    case 0x4e:
+                    case 0x4f:
+                    case 0x50:
+                    case 0x51:
+                    case 0x52:
+                    case 0x53:
+                    case 0x54:
+                    case 0x55:
+                    case 0x56:
+                    case 0x57:
+                    case 0x58:
+                    case 0x59:
+                    case 0x5a:
+                    case 0x5b:
+                    case 0x5d:
+                    case 0x5e:
+                    case 0x5f:
+                    case 0x60:
+                    case 0x61:
+                    case 0x62:
+                    case 0x63:
+                    case 0x64:
+                    case 0x65:
+                    case 0x66:
+                    case 0x67:
+                    case 0x68:
+                    case 0x69:
+                    case 0x6a:
+                    case 0x6b:
+                    case 0x6c:
+                    case 0x6d:
+                    case 0x6e:
+                    case 0x6f:
+                    case 0x70:
+                    case 0x71:
+                    case 0x72:
+                    case 0x73:
+                    case 0x74:
+                    case 0x75:
+                    case 0x76:
+                    case 0x77:
+                    case 0x78:
+                    case 0x79:
+                    case 0x7a:
+                    case 0x7b:
+                    case 0x7c:
+                    case 0x7d:
+                    case 0x7e:
+                    case 0x7f:
+                    {
+                        add(current);
                         break;
                     }
-basic_json_parser_6:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
+
+                    // U+0080..U+07FF: bytes C2..DF 80..BF
+                    case 0xc2:
+                    case 0xc3:
+                    case 0xc4:
+                    case 0xc5:
+                    case 0xc6:
+                    case 0xc7:
+                    case 0xc8:
+                    case 0xc9:
+                    case 0xca:
+                    case 0xcb:
+                    case 0xcc:
+                    case 0xcd:
+                    case 0xce:
+                    case 0xcf:
+                    case 0xd0:
+                    case 0xd1:
+                    case 0xd2:
+                    case 0xd3:
+                    case 0xd4:
+                    case 0xd5:
+                    case 0xd6:
+                    case 0xd7:
+                    case 0xd8:
+                    case 0xd9:
+                    case 0xda:
+                    case 0xdb:
+                    case 0xdc:
+                    case 0xdd:
+                    case 0xde:
+                    case 0xdf:
                     {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yybm[0 + yych] & 32)
-                    {
-                        goto basic_json_parser_6;
-                    }
-                    {
-                        continue;
-                    }
-basic_json_parser_9:
-                    yyaccept = 0;
-                    yych = *(m_marker = ++m_cursor);
-                    if (yych <= 0x1F)
-                    {
-                        goto basic_json_parser_5;
-                    }
-                    if (yych <= 0x7F)
-                    {
-                        goto basic_json_parser_31;
-                    }
-                    if (yych <= 0xC1)
-                    {
-                        goto basic_json_parser_5;
-                    }
-                    if (yych <= 0xF4)
-                    {
-                        goto basic_json_parser_31;
-                    }
-                    goto basic_json_parser_5;
-basic_json_parser_10:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::value_separator;
-                        break;
-                    }
-basic_json_parser_12:
-                    yych = *++m_cursor;
-                    if (yych <= '/')
-                    {
-                        goto basic_json_parser_5;
-                    }
-                    if (yych <= '0')
-                    {
-                        goto basic_json_parser_43;
-                    }
-                    if (yych <= '9')
-                    {
-                        goto basic_json_parser_45;
-                    }
-                    goto basic_json_parser_5;
-basic_json_parser_13:
-                    yyaccept = 1;
-                    yych = *(m_marker = ++m_cursor);
-                    if (yych <= '9')
-                    {
-                        if (yych == '.')
+                        add(current);
+                        get();
+                        if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                         {
-                            goto basic_json_parser_47;
-                        }
-                        if (yych >= '0')
-                        {
-                            goto basic_json_parser_48;
+                            add(current);
+                            continue;
                         }
+
+                        error_message = "invalid string: ill-formed UTF-8 byte";
+                        return token_type::parse_error;
                     }
-                    else
+
+                    // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
+                    case 0xe0:
                     {
-                        if (yych <= 'E')
+                        add(current);
+                        get();
+                        if (JSON_LIKELY(0xa0 <= current and current <= 0xbf))
                         {
-                            if (yych >= 'E')
+                            add(current);
+                            get();
+                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                             {
-                                goto basic_json_parser_51;
+                                add(current);
+                                continue;
                             }
                         }
-                        else
+
+                        error_message = "invalid string: ill-formed UTF-8 byte";
+                        return token_type::parse_error;
+                    }
+
+                    // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
+                    // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
+                    case 0xe1:
+                    case 0xe2:
+                    case 0xe3:
+                    case 0xe4:
+                    case 0xe5:
+                    case 0xe6:
+                    case 0xe7:
+                    case 0xe8:
+                    case 0xe9:
+                    case 0xea:
+                    case 0xeb:
+                    case 0xec:
+                    case 0xee:
+                    case 0xef:
+                    {
+                        add(current);
+                        get();
+                        if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                         {
-                            if (yych == 'e')
+                            add(current);
+                            get();
+                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                             {
-                                goto basic_json_parser_51;
+                                add(current);
+                                continue;
                             }
                         }
+
+                        error_message = "invalid string: ill-formed UTF-8 byte";
+                        return token_type::parse_error;
                     }
-basic_json_parser_14:
+
+                    // U+D000..U+D7FF: bytes ED 80..9F 80..BF
+                    case 0xed:
                     {
-                        last_token_type = token_type::value_unsigned;
-                        break;
-                    }
-basic_json_parser_15:
-                    yyaccept = 1;
-                    m_marker = ++m_cursor;
-                    if ((m_limit - m_cursor) < 3)
-                    {
-                        fill_line_buffer(3);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yybm[0 + yych] & 64)
-                    {
-                        goto basic_json_parser_15;
-                    }
-                    if (yych <= 'D')
-                    {
-                        if (yych == '.')
+                        add(current);
+                        get();
+                        if (JSON_LIKELY(0x80 <= current and current <= 0x9f))
                         {
-                            goto basic_json_parser_47;
-                        }
-                        goto basic_json_parser_14;
-                    }
-                    else
-                    {
-                        if (yych <= 'E')
-                        {
-                            goto basic_json_parser_51;
-                        }
-                        if (yych == 'e')
-                        {
-                            goto basic_json_parser_51;
-                        }
-                        goto basic_json_parser_14;
-                    }
-basic_json_parser_17:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::name_separator;
-                        break;
-                    }
-basic_json_parser_19:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::begin_array;
-                        break;
-                    }
-basic_json_parser_21:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::end_array;
-                        break;
-                    }
-basic_json_parser_23:
-                    yyaccept = 0;
-                    yych = *(m_marker = ++m_cursor);
-                    if (yych == 'a')
-                    {
-                        goto basic_json_parser_52;
-                    }
-                    goto basic_json_parser_5;
-basic_json_parser_24:
-                    yyaccept = 0;
-                    yych = *(m_marker = ++m_cursor);
-                    if (yych == 'u')
-                    {
-                        goto basic_json_parser_53;
-                    }
-                    goto basic_json_parser_5;
-basic_json_parser_25:
-                    yyaccept = 0;
-                    yych = *(m_marker = ++m_cursor);
-                    if (yych == 'r')
-                    {
-                        goto basic_json_parser_54;
-                    }
-                    goto basic_json_parser_5;
-basic_json_parser_26:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::begin_object;
-                        break;
-                    }
-basic_json_parser_28:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::end_object;
-                        break;
-                    }
-basic_json_parser_30:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-basic_json_parser_31:
-                    if (yybm[0 + yych] & 128)
-                    {
-                        goto basic_json_parser_30;
-                    }
-                    if (yych <= 0xE0)
-                    {
-                        if (yych <= '\\')
-                        {
-                            if (yych <= 0x1F)
+                            add(current);
+                            get();
+                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                             {
-                                goto basic_json_parser_32;
-                            }
-                            if (yych <= '"')
-                            {
-                                goto basic_json_parser_33;
-                            }
-                            goto basic_json_parser_35;
-                        }
-                        else
-                        {
-                            if (yych <= 0xC1)
-                            {
-                                goto basic_json_parser_32;
-                            }
-                            if (yych <= 0xDF)
-                            {
-                                goto basic_json_parser_36;
-                            }
-                            goto basic_json_parser_37;
-                        }
-                    }
-                    else
-                    {
-                        if (yych <= 0xEF)
-                        {
-                            if (yych == 0xED)
-                            {
-                                goto basic_json_parser_39;
-                            }
-                            goto basic_json_parser_38;
-                        }
-                        else
-                        {
-                            if (yych <= 0xF0)
-                            {
-                                goto basic_json_parser_40;
-                            }
-                            if (yych <= 0xF3)
-                            {
-                                goto basic_json_parser_41;
-                            }
-                            if (yych <= 0xF4)
-                            {
-                                goto basic_json_parser_42;
+                                add(current);
+                                continue;
                             }
                         }
+
+                        error_message = "invalid string: ill-formed UTF-8 byte";
+                        return token_type::parse_error;
                     }
-basic_json_parser_32:
-                    m_cursor = m_marker;
-                    if (yyaccept <= 1)
+
+                    // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
+                    case 0xf0:
                     {
-                        if (yyaccept == 0)
+                        add(current);
+                        get();
+                        if (JSON_LIKELY(0x90 <= current and current <= 0xbf))
                         {
-                            goto basic_json_parser_5;
-                        }
-                        else
-                        {
-                            goto basic_json_parser_14;
-                        }
-                    }
-                    else
-                    {
-                        if (yyaccept == 2)
-                        {
-                            goto basic_json_parser_44;
-                        }
-                        else
-                        {
-                            goto basic_json_parser_58;
-                        }
-                    }
-basic_json_parser_33:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::value_string;
-                        break;
-                    }
-basic_json_parser_35:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= 'e')
-                    {
-                        if (yych <= '/')
-                        {
-                            if (yych == '"')
+                            add(current);
+                            get();
+                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                             {
-                                goto basic_json_parser_30;
-                            }
-                            if (yych <= '.')
-                            {
-                                goto basic_json_parser_32;
-                            }
-                            goto basic_json_parser_30;
-                        }
-                        else
-                        {
-                            if (yych <= '\\')
-                            {
-                                if (yych <= '[')
+                                add(current);
+                                get();
+                                if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                                 {
-                                    goto basic_json_parser_32;
+                                    add(current);
+                                    continue;
                                 }
-                                goto basic_json_parser_30;
                             }
-                            else
+                        }
+
+                        error_message = "invalid string: ill-formed UTF-8 byte";
+                        return token_type::parse_error;
+                    }
+
+                    // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
+                    case 0xf1:
+                    case 0xf2:
+                    case 0xf3:
+                    {
+                        add(current);
+                        get();
+                        if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
+                        {
+                            add(current);
+                            get();
+                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                             {
-                                if (yych == 'b')
+                                add(current);
+                                get();
+                                if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                                 {
-                                    goto basic_json_parser_30;
+                                    add(current);
+                                    continue;
                                 }
-                                goto basic_json_parser_32;
                             }
                         }
+
+                        error_message = "invalid string: ill-formed UTF-8 byte";
+                        return token_type::parse_error;
                     }
-                    else
+
+                    // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
+                    case 0xf4:
                     {
-                        if (yych <= 'q')
+                        add(current);
+                        get();
+                        if (JSON_LIKELY(0x80 <= current and current <= 0x8f))
                         {
-                            if (yych <= 'f')
+                            add(current);
+                            get();
+                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                             {
-                                goto basic_json_parser_30;
-                            }
-                            if (yych == 'n')
-                            {
-                                goto basic_json_parser_30;
-                            }
-                            goto basic_json_parser_32;
-                        }
-                        else
-                        {
-                            if (yych <= 's')
-                            {
-                                if (yych <= 'r')
+                                add(current);
+                                get();
+                                if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
                                 {
-                                    goto basic_json_parser_30;
+                                    add(current);
+                                    continue;
                                 }
-                                goto basic_json_parser_32;
-                            }
-                            else
-                            {
-                                if (yych <= 't')
-                                {
-                                    goto basic_json_parser_30;
-                                }
-                                if (yych <= 'u')
-                                {
-                                    goto basic_json_parser_55;
-                                }
-                                goto basic_json_parser_32;
                             }
                         }
+
+                        error_message = "invalid string: ill-formed UTF-8 byte";
+                        return token_type::parse_error;
                     }
-basic_json_parser_36:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
+
+                    // remaining bytes (80..C1 and F5..FF) are ill-formed
+                    default:
                     {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= 0x7F)
-                    {
-                        goto basic_json_parser_32;
-                    }
-                    if (yych <= 0xBF)
-                    {
-                        goto basic_json_parser_30;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_37:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= 0x9F)
-                    {
-                        goto basic_json_parser_32;
-                    }
-                    if (yych <= 0xBF)
-                    {
-                        goto basic_json_parser_36;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_38:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= 0x7F)
-                    {
-                        goto basic_json_parser_32;
-                    }
-                    if (yych <= 0xBF)
-                    {
-                        goto basic_json_parser_36;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_39:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= 0x7F)
-                    {
-                        goto basic_json_parser_32;
-                    }
-                    if (yych <= 0x9F)
-                    {
-                        goto basic_json_parser_36;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_40:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= 0x8F)
-                    {
-                        goto basic_json_parser_32;
-                    }
-                    if (yych <= 0xBF)
-                    {
-                        goto basic_json_parser_38;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_41:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= 0x7F)
-                    {
-                        goto basic_json_parser_32;
-                    }
-                    if (yych <= 0xBF)
-                    {
-                        goto basic_json_parser_38;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_42:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= 0x7F)
-                    {
-                        goto basic_json_parser_32;
-                    }
-                    if (yych <= 0x8F)
-                    {
-                        goto basic_json_parser_38;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_43:
-                    yyaccept = 2;
-                    yych = *(m_marker = ++m_cursor);
-                    if (yych <= '9')
-                    {
-                        if (yych == '.')
-                        {
-                            goto basic_json_parser_47;
-                        }
-                        if (yych >= '0')
-                        {
-                            goto basic_json_parser_48;
-                        }
-                    }
-                    else
-                    {
-                        if (yych <= 'E')
-                        {
-                            if (yych >= 'E')
-                            {
-                                goto basic_json_parser_51;
-                            }
-                        }
-                        else
-                        {
-                            if (yych == 'e')
-                            {
-                                goto basic_json_parser_51;
-                            }
-                        }
-                    }
-basic_json_parser_44:
-                    {
-                        last_token_type = token_type::value_integer;
-                        break;
-                    }
-basic_json_parser_45:
-                    yyaccept = 2;
-                    m_marker = ++m_cursor;
-                    if ((m_limit - m_cursor) < 3)
-                    {
-                        fill_line_buffer(3);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= '9')
-                    {
-                        if (yych == '.')
-                        {
-                            goto basic_json_parser_47;
-                        }
-                        if (yych <= '/')
-                        {
-                            goto basic_json_parser_44;
-                        }
-                        goto basic_json_parser_45;
-                    }
-                    else
-                    {
-                        if (yych <= 'E')
-                        {
-                            if (yych <= 'D')
-                            {
-                                goto basic_json_parser_44;
-                            }
-                            goto basic_json_parser_51;
-                        }
-                        else
-                        {
-                            if (yych == 'e')
-                            {
-                                goto basic_json_parser_51;
-                            }
-                            goto basic_json_parser_44;
-                        }
-                    }
-basic_json_parser_47:
-                    yych = *++m_cursor;
-                    if (yych <= '/')
-                    {
-                        goto basic_json_parser_32;
-                    }
-                    if (yych <= '9')
-                    {
-                        goto basic_json_parser_56;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_48:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= '/')
-                    {
-                        goto basic_json_parser_50;
-                    }
-                    if (yych <= '9')
-                    {
-                        goto basic_json_parser_48;
-                    }
-basic_json_parser_50:
-                    {
-                        last_token_type = token_type::parse_error;
-                        break;
-                    }
-basic_json_parser_51:
-                    yych = *++m_cursor;
-                    if (yych <= ',')
-                    {
-                        if (yych == '+')
-                        {
-                            goto basic_json_parser_59;
-                        }
-                        goto basic_json_parser_32;
-                    }
-                    else
-                    {
-                        if (yych <= '-')
-                        {
-                            goto basic_json_parser_59;
-                        }
-                        if (yych <= '/')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= '9')
-                        {
-                            goto basic_json_parser_60;
-                        }
-                        goto basic_json_parser_32;
-                    }
-basic_json_parser_52:
-                    yych = *++m_cursor;
-                    if (yych == 'l')
-                    {
-                        goto basic_json_parser_62;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_53:
-                    yych = *++m_cursor;
-                    if (yych == 'l')
-                    {
-                        goto basic_json_parser_63;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_54:
-                    yych = *++m_cursor;
-                    if (yych == 'u')
-                    {
-                        goto basic_json_parser_64;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_55:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= '@')
-                    {
-                        if (yych <= '/')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= '9')
-                        {
-                            goto basic_json_parser_65;
-                        }
-                        goto basic_json_parser_32;
-                    }
-                    else
-                    {
-                        if (yych <= 'F')
-                        {
-                            goto basic_json_parser_65;
-                        }
-                        if (yych <= '`')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= 'f')
-                        {
-                            goto basic_json_parser_65;
-                        }
-                        goto basic_json_parser_32;
-                    }
-basic_json_parser_56:
-                    yyaccept = 3;
-                    m_marker = ++m_cursor;
-                    if ((m_limit - m_cursor) < 3)
-                    {
-                        fill_line_buffer(3);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= 'D')
-                    {
-                        if (yych <= '/')
-                        {
-                            goto basic_json_parser_58;
-                        }
-                        if (yych <= '9')
-                        {
-                            goto basic_json_parser_56;
-                        }
-                    }
-                    else
-                    {
-                        if (yych <= 'E')
-                        {
-                            goto basic_json_parser_51;
-                        }
-                        if (yych == 'e')
-                        {
-                            goto basic_json_parser_51;
-                        }
-                    }
-basic_json_parser_58:
-                    {
-                        last_token_type = token_type::value_float;
-                        break;
-                    }
-basic_json_parser_59:
-                    yych = *++m_cursor;
-                    if (yych <= '/')
-                    {
-                        goto basic_json_parser_32;
-                    }
-                    if (yych >= ':')
-                    {
-                        goto basic_json_parser_32;
-                    }
-basic_json_parser_60:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= '/')
-                    {
-                        goto basic_json_parser_58;
-                    }
-                    if (yych <= '9')
-                    {
-                        goto basic_json_parser_60;
-                    }
-                    goto basic_json_parser_58;
-basic_json_parser_62:
-                    yych = *++m_cursor;
-                    if (yych == 's')
-                    {
-                        goto basic_json_parser_66;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_63:
-                    yych = *++m_cursor;
-                    if (yych == 'l')
-                    {
-                        goto basic_json_parser_67;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_64:
-                    yych = *++m_cursor;
-                    if (yych == 'e')
-                    {
-                        goto basic_json_parser_69;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_65:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= '@')
-                    {
-                        if (yych <= '/')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= '9')
-                        {
-                            goto basic_json_parser_71;
-                        }
-                        goto basic_json_parser_32;
-                    }
-                    else
-                    {
-                        if (yych <= 'F')
-                        {
-                            goto basic_json_parser_71;
-                        }
-                        if (yych <= '`')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= 'f')
-                        {
-                            goto basic_json_parser_71;
-                        }
-                        goto basic_json_parser_32;
-                    }
-basic_json_parser_66:
-                    yych = *++m_cursor;
-                    if (yych == 'e')
-                    {
-                        goto basic_json_parser_72;
-                    }
-                    goto basic_json_parser_32;
-basic_json_parser_67:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::literal_null;
-                        break;
-                    }
-basic_json_parser_69:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::literal_true;
-                        break;
-                    }
-basic_json_parser_71:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= '@')
-                    {
-                        if (yych <= '/')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= '9')
-                        {
-                            goto basic_json_parser_74;
-                        }
-                        goto basic_json_parser_32;
-                    }
-                    else
-                    {
-                        if (yych <= 'F')
-                        {
-                            goto basic_json_parser_74;
-                        }
-                        if (yych <= '`')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= 'f')
-                        {
-                            goto basic_json_parser_74;
-                        }
-                        goto basic_json_parser_32;
-                    }
-basic_json_parser_72:
-                    ++m_cursor;
-                    {
-                        last_token_type = token_type::literal_false;
-                        break;
-                    }
-basic_json_parser_74:
-                    ++m_cursor;
-                    if (m_limit <= m_cursor)
-                    {
-                        fill_line_buffer(1);    // LCOV_EXCL_LINE
-                    }
-                    yych = *m_cursor;
-                    if (yych <= '@')
-                    {
-                        if (yych <= '/')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= '9')
-                        {
-                            goto basic_json_parser_30;
-                        }
-                        goto basic_json_parser_32;
-                    }
-                    else
-                    {
-                        if (yych <= 'F')
-                        {
-                            goto basic_json_parser_30;
-                        }
-                        if (yych <= '`')
-                        {
-                            goto basic_json_parser_32;
-                        }
-                        if (yych <= 'f')
-                        {
-                            goto basic_json_parser_30;
-                        }
-                        goto basic_json_parser_32;
+                        error_message = "invalid string: ill-formed UTF-8 byte";
+                        return token_type::parse_error;
                     }
                 }
-
             }
+        }
 
-            return last_token_type;
+        static void strtof(float& f, const char* str, char** endptr) noexcept
+        {
+            f = std::strtof(str, endptr);
+        }
+
+        static void strtof(double& f, const char* str, char** endptr) noexcept
+        {
+            f = std::strtod(str, endptr);
+        }
+
+        static void strtof(long double& f, const char* str, char** endptr) noexcept
+        {
+            f = std::strtold(str, endptr);
         }
 
         /*!
-        @brief append data from the stream to the line buffer
+        @brief scan a number literal
 
-        This function is called by the scan() function when the end of the
-        buffer (`m_limit`) is reached and the `m_cursor` pointer cannot be
-        incremented without leaving the limits of the line buffer. Note re2c
-        decides when to call this function.
+        This function scans a string according to Sect. 6 of RFC 7159.
 
-        If the lexer reads from contiguous storage, there is no trailing null
-        byte. Therefore, this function must make sure to add these padding
-        null bytes.
+        The function is realized with a deterministic finite state machine
+        derived from the grammar described in RFC 7159. Starting in state
+        "init", the input is read and used to determined the next state. Only
+        state "done" accepts the number. State "error" is a trap state to model
+        errors. In the table below, "anything" means any character but the ones
+        listed before.
 
-        If the lexer reads from an input stream, this function reads the next
-        line of the input.
+        state    | 0        | 1-9      | e E      | +       | -       | .        | anything
+        ---------|----------|----------|----------|---------|---------|----------|-----------
+        init     | zero     | any1     | [error]  | [error] | minus   | [error]  | [error]
+        minus    | zero     | any1     | [error]  | [error] | [error] | [error]  | [error]
+        zero     | done     | done     | exponent | done    | done    | decimal1 | done
+        any1     | any1     | any1     | exponent | done    | done    | decimal1 | done
+        decimal1 | decimal2 | [error]  | [error]  | [error] | [error] | [error]  | [error]
+        decimal2 | decimal2 | decimal2 | exponent | done    | done    | done     | done
+        exponent | any2     | any2     | [error]  | sign    | sign    | [error]  | [error]
+        sign     | any2     | any2     | [error]  | [error] | [error] | [error]  | [error]
+        any2     | any2     | any2     | done     | done    | done    | done     | done
 
-        @pre
-            p p p p p p u u u u u x . . . . . .
-            ^           ^       ^   ^
-            m_content   m_start |   m_limit
-                                m_cursor
+        The state machine is realized with one label per state (prefixed with
+        "scan_number_") and `goto` statements between them. The state machine
+        contains cycles, but any cycle can be left when EOF is read. Therefore,
+        the function is guaranteed to terminate.
 
-        @post
-            u u u u u x x x x x x x . . . . . .
-            ^       ^               ^
-            |       m_cursor        m_limit
-            m_start
-            m_content
+        During scanning, the read bytes are stored in yytext. This string is
+        then converted to a signed integer, an unsigned integer, or a
+        floating-point number.
+
+        @return token_type::value_unsigned, token_type::value_integer, or
+                token_type::value_float if number could be successfully scanned,
+                token_type::parse_error otherwise
+
+        @note The scanner is independent of the current locale. Internally, the
+              locale's decimal point is used instead of `.` to work with the
+              locale-dependent converters.
         */
-        void fill_line_buffer(size_t n = 0)
+        token_type scan_number()
         {
-            // if line buffer is used, m_content points to its data
-            assert(m_line_buffer.empty()
-                   or m_content == reinterpret_cast<const lexer_char_t*>(m_line_buffer.data()));
+            // reset yytext to store the number's bytes
+            reset();
 
-            // if line buffer is used, m_limit is set past the end of its data
-            assert(m_line_buffer.empty()
-                   or m_limit == m_content + m_line_buffer.size());
+            // the type of the parsed number; initially set to unsigned; will be
+            // changed if minus sign, decimal point or exponent is read
+            token_type number_type = token_type::value_unsigned;
 
-            // pointer relationships
-            assert(m_content <= m_start);
-            assert(m_start <= m_cursor);
-            assert(m_cursor <= m_limit);
-            assert(m_marker == nullptr or m_marker  <= m_limit);
-
-            // number of processed characters (p)
-            const auto num_processed_chars = static_cast<size_t>(m_start - m_content);
-            // offset for m_marker wrt. to m_start
-            const auto offset_marker = (m_marker == nullptr) ? 0 : m_marker - m_start;
-            // number of unprocessed characters (u)
-            const auto offset_cursor = m_cursor - m_start;
-
-            // no stream is used or end of file is reached
-            if (m_stream == nullptr or m_stream->eof())
+            // state (init): we just found out we need to scan a number
+            switch (current)
             {
-                // m_start may or may not be pointing into m_line_buffer at
-                // this point. We trust the standard library to do the right
-                // thing. See http://stackoverflow.com/q/28142011/266378
-                m_line_buffer.assign(m_start, m_limit);
-
-                // append n characters to make sure that there is sufficient
-                // space between m_cursor and m_limit
-                m_line_buffer.append(1, '\x00');
-                if (n > 0)
+                case '-':
                 {
-                    m_line_buffer.append(n - 1, '\x01');
+                    add(current);
+                    goto scan_number_minus;
+                }
+
+                case '0':
+                {
+                    add(current);
+                    goto scan_number_zero;
+                }
+
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                case '8':
+                case '9':
+                {
+                    add(current);
+                    goto scan_number_any1;
+                }
+
+                default:
+                {
+                    // all other characters are rejected outside scan_number()
+                    assert(false);  // LCOV_EXCL_LINE
                 }
             }
-            else
-            {
-                // delete processed characters from line buffer
-                m_line_buffer.erase(0, num_processed_chars);
-                // read next line from input stream
-                m_line_buffer_tmp.clear();
-                std::getline(*m_stream, m_line_buffer_tmp, '\n');
 
-                // add line with newline symbol to the line buffer
-                m_line_buffer += m_line_buffer_tmp;
-                m_line_buffer.push_back('\n');
+scan_number_minus:
+            // state: we just parsed a leading minus sign
+            number_type = token_type::value_integer;
+            switch (get())
+            {
+                case '0':
+                {
+                    add(current);
+                    goto scan_number_zero;
+                }
+
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                case '8':
+                case '9':
+                {
+                    add(current);
+                    goto scan_number_any1;
+                }
+
+                default:
+                {
+                    error_message = "invalid number; expected digit after '-'";
+                    return token_type::parse_error;
+                }
             }
 
-            // set pointers
-            m_content = reinterpret_cast<const lexer_char_t*>(m_line_buffer.data());
-            assert(m_content != nullptr);
-            m_start  = m_content;
-            m_marker = m_start + offset_marker;
-            m_cursor = m_start + offset_cursor;
-            m_limit  = m_start + m_line_buffer.size();
-        }
+scan_number_zero:
+            // state: we just parse a zero (maybe with a leading minus sign)
+            switch (get())
+            {
+                case '.':
+                {
+                    add(decimal_point_char);
+                    goto scan_number_decimal1;
+                }
 
-        /// return string representation of last read token
-        string_t get_token_string() const
-        {
-            assert(m_start != nullptr);
-            return string_t(reinterpret_cast<typename string_t::const_pointer>(m_start),
-                            static_cast<size_t>(m_cursor - m_start));
+                case 'e':
+                case 'E':
+                {
+                    add(current);
+                    goto scan_number_exponent;
+                }
+
+                default:
+                {
+                    goto scan_number_done;
+                }
+            }
+
+scan_number_any1:
+            // state: we just parsed a number 0-9 (maybe with a leading minus sign)
+            switch (get())
+            {
+                case '0':
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                case '8':
+                case '9':
+                {
+                    add(current);
+                    goto scan_number_any1;
+                }
+
+                case '.':
+                {
+                    add(decimal_point_char);
+                    goto scan_number_decimal1;
+                }
+
+                case 'e':
+                case 'E':
+                {
+                    add(current);
+                    goto scan_number_exponent;
+                }
+
+                default:
+                {
+                    goto scan_number_done;
+                }
+            }
+
+scan_number_decimal1:
+            // state: we just parsed a decimal point
+            number_type = token_type::value_float;
+            switch (get())
+            {
+                case '0':
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                case '8':
+                case '9':
+                {
+                    add(current);
+                    goto scan_number_decimal2;
+                }
+
+                default:
+                {
+                    error_message = "invalid number; expected digit after '.'";
+                    return token_type::parse_error;
+                }
+            }
+
+scan_number_decimal2:
+            // we just parsed at least one number after a decimal point
+            switch (get())
+            {
+                case '0':
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                case '8':
+                case '9':
+                {
+                    add(current);
+                    goto scan_number_decimal2;
+                }
+
+                case 'e':
+                case 'E':
+                {
+                    add(current);
+                    goto scan_number_exponent;
+                }
+
+                default:
+                {
+                    goto scan_number_done;
+                }
+            }
+
+scan_number_exponent:
+            // we just parsed an exponent
+            number_type = token_type::value_float;
+            switch (get())
+            {
+                case '+':
+                case '-':
+                {
+                    add(current);
+                    goto scan_number_sign;
+                }
+
+                case '0':
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                case '8':
+                case '9':
+                {
+                    add(current);
+                    goto scan_number_any2;
+                }
+
+                default:
+                {
+                    error_message = "invalid number; expected '+', '-', or digit after exponent";
+                    return token_type::parse_error;
+                }
+            }
+
+scan_number_sign:
+            // we just parsed an exponent sign
+            switch (get())
+            {
+                case '0':
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                case '8':
+                case '9':
+                {
+                    add(current);
+                    goto scan_number_any2;
+                }
+
+                default:
+                {
+                    error_message = "invalid number; expected digit after exponent sign";
+                    return token_type::parse_error;
+                }
+            }
+
+scan_number_any2:
+            // we just parsed a number after the exponent or exponent sign
+            switch (get())
+            {
+                case '0':
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                case '8':
+                case '9':
+                {
+                    add(current);
+                    goto scan_number_any2;
+                }
+
+                default:
+                {
+                    goto scan_number_done;
+                }
+            }
+
+scan_number_done:
+            // unget the character after the number (we only read it to know
+            // that we are done scanning a number)
+            --chars_read;
+            next_unget = true;
+
+            // terminate token
+            add('\0');
+            --yylen;
+
+            // try to parse integers first and fall back to floats
+            if (number_type == token_type::value_unsigned)
+            {
+                char* endptr = nullptr;
+                errno = 0;
+                const auto x = std::strtoull(yytext.data(), &endptr, 10);
+
+                // we checked the number format before
+                assert(endptr == yytext.data() + yylen);
+
+                if (errno == 0)
+                {
+                    value_unsigned = static_cast<number_unsigned_t>(x);
+                    if (value_unsigned == x)
+                    {
+                        return token_type::value_unsigned;
+                    }
+                }
+            }
+            else if (number_type == token_type::value_integer)
+            {
+                char* endptr = nullptr;
+                errno = 0;
+                const auto x = std::strtoll(yytext.data(), &endptr, 10);
+
+                // we checked the number format before
+                assert(endptr == yytext.data() + yylen);
+
+                if (errno == 0)
+                {
+                    value_integer = static_cast<number_integer_t>(x);
+                    if (value_integer == x)
+                    {
+                        return token_type::value_integer;
+                    }
+                }
+            }
+
+            // this code is reached if we parse a floating-point number or if
+            // an integer conversion above failed
+            strtof(value_float, yytext.data(), nullptr);
+            return token_type::value_float;
         }
 
         /*!
-        @brief return string value for string tokens
-
-        The function iterates the characters between the opening and closing
-        quotes of the string value. The complete string is the range
-        [m_start,m_cursor). Consequently, we iterate from m_start+1 to
-        m_cursor-1.
-
-        We differentiate two cases:
-
-        1. Escaped characters. In this case, a new character is constructed
-           according to the nature of the escape. Some escapes create new
-           characters (e.g., `"\\n"` is replaced by `"\n"`), some are copied
-           as is (e.g., `"\\\\"`). Furthermore, Unicode escapes of the shape
-           `"\\uxxxx"` need special care. In this case, to_unicode takes care
-           of the construction of the values.
-        2. Unescaped characters are copied as is.
-
-        @pre `m_cursor - m_start >= 2`, meaning the length of the last token
-        is at least 2 bytes which is trivially true for any string (which
-        consists of at least two quotes).
-
-            " c1 c2 c3 ... "
-            ^                ^
-            m_start          m_cursor
-
-        @complexity Linear in the length of the string.\n
-
-        Lemma: The loop body will always terminate.\n
-
-        Proof (by contradiction): Assume the loop body does not terminate. As
-        the loop body does not contain another loop, one of the called
-        functions must never return. The called functions are `std::strtoul`
-        and to_unicode. Neither function can loop forever, so the loop body
-        will never loop forever which contradicts the assumption that the loop
-        body does not terminate, q.e.d.\n
-
-        Lemma: The loop condition for the for loop is eventually false.\n
-
-        Proof (by contradiction): Assume the loop does not terminate. Due to
-        the above lemma, this can only be due to a tautological loop
-        condition; that is, the loop condition i < m_cursor - 1 must always be
-        true. Let x be the change of i for any loop iteration. Then
-        m_start + 1 + x < m_cursor - 1 must hold to loop indefinitely. This
-        can be rephrased to m_cursor - m_start - 2 > x. With the
-        precondition, we x <= 0, meaning that the loop condition holds
-        indefinitely if i is always decreased. However, observe that the value
-        of i is strictly increasing with each iteration, as it is incremented
-        by 1 in the iteration expression and never decremented inside the loop
-        body. Hence, the loop condition will eventually be false which
-        contradicts the assumption that the loop condition is a tautology,
-        q.e.d.
-
-        @return string value of current token without opening and closing
-        quotes
-        @throw std::out_of_range if to_unicode fails
+        @param[in] literal_text  the literal text to expect
+        @param[in] length        the length of the passed literal text
+        @param[in] return_type   the token type to return on success
         */
-        string_t get_string() const
+        token_type scan_literal(const char* literal_text, const size_t length,
+                                token_type return_type)
         {
-            assert(m_cursor - m_start >= 2);
-
-            string_t result;
-            result.reserve(static_cast<size_t>(m_cursor - m_start - 2));
-
-            // iterate the result between the quotes
-            for (const lexer_char_t* i = m_start + 1; i < m_cursor - 1; ++i)
+            assert(current == literal_text[0]);
+            for (size_t i = 1; i < length; ++i)
             {
-                // find next escape character
-                auto e = std::find(i, m_cursor - 1, '\\');
-                if (e != i)
+                if (JSON_UNLIKELY(get() != literal_text[i]))
                 {
-                    // see https://github.com/nlohmann/json/issues/365#issuecomment-262874705
-                    for (auto k = i; k < e; k++)
-                    {
-                        result.push_back(static_cast<typename string_t::value_type>(*k));
-                    }
-                    i = e - 1; // -1 because of ++i
+                    error_message = "invalid literal";
+                    return token_type::parse_error;
+                }
+            }
+            return return_type;
+        }
+
+        /////////////////////
+        // input management
+        /////////////////////
+
+        /// reset yytext
+        void reset() noexcept
+        {
+            yylen = 0;
+            start_pos = chars_read - 1;
+        }
+
+        /// get a character from the input
+        int get()
+        {
+            ++chars_read;
+            return next_unget
+                   ? (next_unget = false, current)
+                   : (current = ia->get_character());
+        }
+
+        /// add a character to yytext
+        void add(int c)
+        {
+            // resize yytext if necessary; this condition is deemed unlikely,
+            // because we start with a 1024-byte buffer
+            if (JSON_UNLIKELY((yylen + 1 > yytext.capacity())))
+            {
+                yytext.resize(2 * yytext.capacity(), '\0');
+            }
+            assert(yylen < yytext.size());
+            yytext[yylen++] = static_cast<char>(c);
+        }
+
+      public:
+        /////////////////////
+        // value getters
+        /////////////////////
+
+        /// return integer value
+        constexpr number_integer_t get_number_integer() const noexcept
+        {
+            return value_integer;
+        }
+
+        /// return unsigned integer value
+        constexpr number_unsigned_t get_number_unsigned() const noexcept
+        {
+            return value_unsigned;
+        }
+
+        /// return floating-point value
+        constexpr number_float_t get_number_float() const noexcept
+        {
+            return value_float;
+        }
+
+        /// return string value
+        const std::string get_string()
+        {
+            // yytext cannot be returned as char*, because it may contain a
+            // null byte (parsed as "\u0000")
+            return std::string(yytext.data(), yylen);
+        }
+
+        /////////////////////
+        // diagnostics
+        /////////////////////
+
+        /// return position of last read token
+        constexpr size_t get_position() const noexcept
+        {
+            return chars_read;
+        }
+
+        /// return the last read token (for errors only)
+        std::string get_token_string() const
+        {
+            // get the raw byte sequence of the last token
+            std::string s = ia->read(start_pos, chars_read - start_pos);
+
+            // escape control characters
+            std::string result;
+            for (auto c : s)
+            {
+                if (c == '\0' or c == std::char_traits<char>::eof())
+                {
+                    // ignore EOF
+                    continue;
+                }
+                else if ('\x00' <= c and c <= '\x1f')
+                {
+                    // escape control characters
+                    std::stringstream ss;
+                    ss << "<U+" << std::setw(4) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(c) << ">";
+                    result += ss.str();
                 }
                 else
                 {
-                    // processing escaped character
-                    // read next character
-                    ++i;
-
-                    switch (*i)
-                    {
-                        // the default escapes
-                        case 't':
-                        {
-                            result += "\t";
-                            break;
-                        }
-                        case 'b':
-                        {
-                            result += "\b";
-                            break;
-                        }
-                        case 'f':
-                        {
-                            result += "\f";
-                            break;
-                        }
-                        case 'n':
-                        {
-                            result += "\n";
-                            break;
-                        }
-                        case 'r':
-                        {
-                            result += "\r";
-                            break;
-                        }
-                        case '\\':
-                        {
-                            result += "\\";
-                            break;
-                        }
-                        case '/':
-                        {
-                            result += "/";
-                            break;
-                        }
-                        case '"':
-                        {
-                            result += "\"";
-                            break;
-                        }
-
-                        // unicode
-                        case 'u':
-                        {
-                            // get code xxxx from uxxxx
-                            auto codepoint = std::strtoul(std::string(reinterpret_cast<typename string_t::const_pointer>(i + 1),
-                                                          4).c_str(), nullptr, 16);
-
-                            // check if codepoint is a high surrogate
-                            if (codepoint >= 0xD800 and codepoint <= 0xDBFF)
-                            {
-                                // make sure there is a subsequent unicode
-                                if ((i + 6 >= m_limit) or * (i + 5) != '\\' or * (i + 6) != 'u')
-                                {
-                                    JSON_THROW(std::invalid_argument("missing low surrogate"));
-                                }
-
-                                // get code yyyy from uxxxx\uyyyy
-                                auto codepoint2 = std::strtoul(std::string(reinterpret_cast<typename string_t::const_pointer>
-                                                               (i + 7), 4).c_str(), nullptr, 16);
-                                result += to_unicode(codepoint, codepoint2);
-                                // skip the next 10 characters (xxxx\uyyyy)
-                                i += 10;
-                            }
-                            else if (codepoint >= 0xDC00 and codepoint <= 0xDFFF)
-                            {
-                                // we found a lone low surrogate
-                                JSON_THROW(std::invalid_argument("missing high surrogate"));
-                            }
-                            else
-                            {
-                                // add unicode character(s)
-                                result += to_unicode(codepoint);
-                                // skip the next four characters (xxxx)
-                                i += 4;
-                            }
-                            break;
-                        }
-                    }
+                    // add character as is
+                    result.append(1, c);
                 }
             }
 
             return result;
         }
 
-
-        /*!
-        @brief parse string into a built-in arithmetic type as if the current
-               locale is POSIX.
-
-        @note in floating-point case strtod may parse past the token's end -
-              this is not an error
-
-        @note any leading blanks are not handled
-        */
-        struct strtonum
+        /// return syntax error message
+        constexpr const char* get_error_message() const noexcept
         {
-          public:
-            strtonum(const char* start, const char* end)
-                : m_start(start), m_end(end)
-            {}
+            return error_message;
+        }
 
-            /*!
-            @return true iff parsed successfully as number of type T
+        /////////////////////
+        // actual scanner
+        /////////////////////
 
-            @param[in,out] val shall contain parsed value, or undefined value
-            if could not parse
-            */
-            template<typename T, typename = typename std::enable_if<std::is_arithmetic<T>::value>::type>
-            bool to(T& val) const
-            {
-                return parse(val, std::is_integral<T>());
-            }
-
-          private:
-            const char* const m_start = nullptr;
-            const char* const m_end = nullptr;
-
-            // floating-point conversion
-
-            // overloaded wrappers for strtod/strtof/strtold
-            // that will be called from parse<floating_point_t>
-            static void strtof(float& f, const char* str, char** endptr)
-            {
-                f = std::strtof(str, endptr);
-            }
-
-            static void strtof(double& f, const char* str, char** endptr)
-            {
-                f = std::strtod(str, endptr);
-            }
-
-            static void strtof(long double& f, const char* str, char** endptr)
-            {
-                f = std::strtold(str, endptr);
-            }
-
-            template<typename T>
-            bool parse(T& value, /*is_integral=*/std::false_type) const
-            {
-                // replace decimal separator with locale-specific version,
-                // when necessary; data will point to either the original
-                // string, or buf, or tempstr containing the fixed string.
-                std::string tempstr;
-                std::array<char, 64> buf;
-                const size_t len = static_cast<size_t>(m_end - m_start);
-
-                // lexer will reject empty numbers
-                assert(len > 0);
-
-                // since dealing with strtod family of functions, we're
-                // getting the decimal point char from the C locale facilities
-                // instead of C++'s numpunct facet of the current std::locale
-                const auto loc = localeconv();
-                assert(loc != nullptr);
-                const char decimal_point_char = (loc->decimal_point == nullptr) ? '.' : loc->decimal_point[0];
-
-                const char* data = m_start;
-
-                if (decimal_point_char != '.')
-                {
-                    const size_t ds_pos = static_cast<size_t>(std::find(m_start, m_end, '.') - m_start);
-
-                    if (ds_pos != len)
-                    {
-                        // copy the data into the local buffer or tempstr, if
-                        // buffer is too small; replace decimal separator, and
-                        // update data to point to the modified bytes
-                        if ((len + 1) < buf.size())
-                        {
-                            std::copy(m_start, m_end, buf.begin());
-                            buf[len] = 0;
-                            buf[ds_pos] = decimal_point_char;
-                            data = buf.data();
-                        }
-                        else
-                        {
-                            tempstr.assign(m_start, m_end);
-                            tempstr[ds_pos] = decimal_point_char;
-                            data = tempstr.c_str();
-                        }
-                    }
-                }
-
-                char* endptr = nullptr;
-                value = 0;
-                // this calls appropriate overload depending on T
-                strtof(value, data, &endptr);
-
-                // parsing was successful iff strtof parsed exactly the number
-                // of characters determined by the lexer (len)
-                const bool ok = (endptr == (data + len));
-
-                if (ok and (value == static_cast<T>(0.0)) and (*data == '-'))
-                {
-                    // some implementations forget to negate the zero
-                    value = -0.0;
-                }
-
-                return ok;
-            }
-
-            // integral conversion
-
-            signed long long parse_integral(char** endptr, /*is_signed*/std::true_type) const
-            {
-                return std::strtoll(m_start, endptr, 10);
-            }
-
-            unsigned long long parse_integral(char** endptr, /*is_signed*/std::false_type) const
-            {
-                return std::strtoull(m_start, endptr, 10);
-            }
-
-            template<typename T>
-            bool parse(T& value, /*is_integral=*/std::true_type) const
-            {
-                char* endptr = nullptr;
-                errno = 0; // these are thread-local
-                const auto x = parse_integral(&endptr, std::is_signed<T>());
-
-                // called right overload?
-                static_assert(std::is_signed<T>() == std::is_signed<decltype(x)>(), "");
-
-                value = static_cast<T>(x);
-
-                return (x == static_cast<decltype(x)>(value)) // x fits into destination T
-                       and (x < 0) == (value < 0)             // preserved sign
-                       //and ((x != 0) or is_integral())        // strto[u]ll did nto fail
-                       and (errno == 0)                       // strto[u]ll did not overflow
-                       and (m_start < m_end)                  // token was not empty
-                       and (endptr == m_end);                 // parsed entire token exactly
-            }
-        };
-
-        /*!
-        @brief return number value for number tokens
-
-        This function translates the last token into the most appropriate
-        number type (either integer, unsigned integer or floating point),
-        which is passed back to the caller via the result parameter.
-
-        integral numbers that don't fit into the the range of the respective
-        type are parsed as number_float_t
-
-        floating-point values do not satisfy std::isfinite predicate
-        are converted to value_t::null
-
-        throws if the entire string [m_start .. m_cursor) cannot be
-        interpreted as a number
-
-        @param[out] result  @ref basic_json object to receive the number.
-        @param[in]  token   the type of the number token
-        */
-        bool get_number(basic_json& result, const token_type token) const
+        token_type scan()
         {
-            assert(m_start != nullptr);
-            assert(m_start < m_cursor);
-            assert((token == token_type::value_unsigned) or
-                   (token == token_type::value_integer) or
-                   (token == token_type::value_float));
-
-            strtonum num_converter(reinterpret_cast<const char*>(m_start),
-                                   reinterpret_cast<const char*>(m_cursor));
-
-            switch (token)
+            // read next character and ignore whitespace
+            do
             {
-                case lexer::token_type::value_unsigned:
-                {
-                    number_unsigned_t val;
-                    if (num_converter.to(val))
-                    {
-                        // parsing successful
-                        result.m_type = value_t::number_unsigned;
-                        result.m_value = val;
-                        return true;
-                    }
-                    break;
-                }
+                get();
+            }
+            while (current == ' ' or current == '\t' or current == '\n' or current == '\r');
 
-                case lexer::token_type::value_integer:
-                {
-                    number_integer_t val;
-                    if (num_converter.to(val))
-                    {
-                        // parsing successful
-                        result.m_type = value_t::number_integer;
-                        result.m_value = val;
-                        return true;
-                    }
-                    break;
-                }
+            switch (current)
+            {
+                // structural characters
+                case '[':
+                    return token_type::begin_array;
+                case ']':
+                    return token_type::end_array;
+                case '{':
+                    return token_type::begin_object;
+                case '}':
+                    return token_type::end_object;
+                case ':':
+                    return token_type::name_separator;
+                case ',':
+                    return token_type::value_separator;
 
+                // literals
+                case 't':
+                    return scan_literal("true", 4, token_type::literal_true);
+                case 'f':
+                    return scan_literal("false", 5, token_type::literal_false);
+                case 'n':
+                    return scan_literal("null", 4, token_type::literal_null);
+
+                // string
+                case '\"':
+                    return scan_string();
+
+                // number
+                case '-':
+                case '0':
+                case '1':
+                case '2':
+                case '3':
+                case '4':
+                case '5':
+                case '6':
+                case '7':
+                case '8':
+                case '9':
+                    return scan_number();
+
+                // end of input (the null byte is needed when parsing from
+                // string literals)
+                case '\0':
+                case std::char_traits<char>::eof():
+                    return token_type::end_of_input;
+
+                // error
                 default:
-                {
-                    break;
-                }
+                    error_message = "invalid literal";
+                    return token_type::parse_error;
             }
-
-            // parse float (either explicitly or because a previous conversion
-            // failed)
-            number_float_t val;
-            if (num_converter.to(val))
-            {
-                // parsing successful
-                result.m_type = value_t::number_float;
-                result.m_value = val;
-
-                // replace infinity and NAN by null
-                if (not std::isfinite(result.m_value.number_float))
-                {
-                    result.m_type  = value_t::null;
-                    result.m_value = basic_json::json_value();
-                }
-
-                return true;
-            }
-
-            // couldn't parse number in any format
-            return false;
         }
 
       private:
-        /// optional input stream
-        std::istream* m_stream = nullptr;
-        /// line buffer buffer for m_stream
-        string_t m_line_buffer {};
-        /// used for filling m_line_buffer
-        string_t m_line_buffer_tmp {};
-        /// the buffer pointer
-        const lexer_char_t* m_content = nullptr;
-        /// pointer to the beginning of the current symbol
-        const lexer_char_t* m_start = nullptr;
-        /// pointer for backtracking information
-        const lexer_char_t* m_marker = nullptr;
-        /// pointer to the current symbol
-        const lexer_char_t* m_cursor = nullptr;
-        /// pointer to the end of the buffer
-        const lexer_char_t* m_limit = nullptr;
-        /// the last token type
-        token_type last_token_type = token_type::end_of_input;
+        /// input adapter
+        input_adapter_t ia = nullptr;
+
+        /// the current character
+        int current = std::char_traits<char>::eof();
+
+        /// whether get() should return the last character again
+        bool next_unget = false;
+
+        /// the number of characters read
+        size_t chars_read = 0;
+        /// the start position of the current token
+        size_t start_pos = 0;
+
+        /// buffer for variable-length tokens (numbers, strings)
+        std::vector<char> yytext = std::vector<char>(1024, '\0');
+        /// current index in yytext
+        size_t yylen = 0;
+
+        /// a description of occurred lexer errors
+        const char* error_message = "";
+
+        // number values
+        number_integer_t value_integer = 0;
+        number_unsigned_t value_unsigned = 0;
+        number_float_t value_float = 0;
+
+        /// the decimal point
+        const char decimal_point_char = '.';
     };
 
     /*!
@@ -11353,30 +12623,23 @@ basic_json_parser_74:
     class parser
     {
       public:
-        /// a parser reading from a string literal
-        parser(const char* buff, const parser_callback_t cb = nullptr)
-            : callback(cb),
-              m_lexer(reinterpret_cast<const typename lexer::lexer_char_t*>(buff), std::strlen(buff))
+        /// a parser reading from an input adapter
+        explicit parser(input_adapter_t adapter,
+                        const parser_callback_t cb = nullptr)
+            : callback(cb), m_lexer(adapter)
         {}
 
-        /// a parser reading from an input stream
-        parser(std::istream& is, const parser_callback_t cb = nullptr)
-            : callback(cb), m_lexer(is)
-        {}
+        /*!
+        @brief public parser interface
 
-        /// a parser reading from an iterator range with contiguous storage
-        template<class IteratorType, typename std::enable_if<
-                     std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value
-                     , int>::type
-                 = 0>
-        parser(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr)
-            : callback(cb),
-              m_lexer(reinterpret_cast<const typename lexer::lexer_char_t*>(&(*first)),
-                      static_cast<size_t>(std::distance(first, last)))
-        {}
+        @param[in] strict  whether to expect the last token to be EOF
+        @return parsed JSON value
 
-        /// public parser interface
-        basic_json parse()
+        @throw parse_error.101 in case of an unexpected token
+        @throw parse_error.102 if to_unicode fails or surrogate error
+        @throw parse_error.103 if to_unicode fails
+        */
+        basic_json parse(const bool strict = true)
         {
             // read first token
             get_token();
@@ -11384,15 +12647,48 @@ basic_json_parser_74:
             basic_json result = parse_internal(true);
             result.assert_invariant();
 
-            expect(lexer::token_type::end_of_input);
+            if (strict)
+            {
+                get_token();
+                expect(lexer::token_type::end_of_input);
+            }
 
             // return parser result and replace it with null in case the
             // top-level value was discarded by the callback function
             return result.is_discarded() ? basic_json() : std::move(result);
         }
 
+        /*!
+        @brief public accept interface
+
+        @param[in] strict  whether to expect the last token to be EOF
+        @return whether the input is a proper JSON text
+        */
+        bool accept(const bool strict = true)
+        {
+            // read first token
+            get_token();
+
+            if (not accept_internal())
+            {
+                return false;
+            }
+
+            if (strict and get_token() != lexer::token_type::end_of_input)
+            {
+                return false;
+            }
+
+            return true;
+        }
+
       private:
-        /// the actual parser
+        /*!
+        @brief the actual parser
+        @throw parse_error.101 in case of an unexpected token
+        @throw parse_error.102 if to_unicode fails or surrogate error
+        @throw parse_error.103 if to_unicode fails
+        */
         basic_json parse_internal(bool keep)
         {
             auto result = basic_json(value_t::discarded);
@@ -11415,7 +12711,6 @@ basic_json_parser_74:
                     // closing } -> we are done
                     if (last_token == lexer::token_type::end_object)
                     {
-                        get_token();
                         if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
                         {
                             result = basic_json(value_t::discarded);
@@ -11423,18 +12718,9 @@ basic_json_parser_74:
                         return result;
                     }
 
-                    // no comma is expected here
-                    unexpect(lexer::token_type::value_separator);
-
-                    // otherwise: parse key-value pairs
-                    do
+                    // parse values
+                    while (true)
                     {
-                        // ugly, but could be fixed with loop reorganization
-                        if (last_token == lexer::token_type::value_separator)
-                        {
-                            get_token();
-                        }
-
                         // store key
                         expect(lexer::token_type::value_string);
                         const auto key = m_lexer.get_string();
@@ -11464,12 +12750,20 @@ basic_json_parser_74:
                         {
                             result[key] = std::move(value);
                         }
-                    }
-                    while (last_token == lexer::token_type::value_separator);
 
-                    // closing }
-                    expect(lexer::token_type::end_object);
-                    get_token();
+                        // comma -> next value
+                        get_token();
+                        if (last_token == lexer::token_type::value_separator)
+                        {
+                            get_token();
+                            continue;
+                        }
+
+                        // closing }
+                        expect(lexer::token_type::end_object);
+                        break;
+                    }
+
                     if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
                     {
                         result = basic_json(value_t::discarded);
@@ -11494,7 +12788,6 @@ basic_json_parser_74:
                     // closing ] -> we are done
                     if (last_token == lexer::token_type::end_array)
                     {
-                        get_token();
                         if (callback and not callback(--depth, parse_event_t::array_end, result))
                         {
                             result = basic_json(value_t::discarded);
@@ -11502,30 +12795,29 @@ basic_json_parser_74:
                         return result;
                     }
 
-                    // no comma is expected here
-                    unexpect(lexer::token_type::value_separator);
-
-                    // otherwise: parse values
-                    do
+                    // parse values
+                    while (true)
                     {
-                        // ugly, but could be fixed with loop reorganization
-                        if (last_token == lexer::token_type::value_separator)
-                        {
-                            get_token();
-                        }
-
                         // parse value
                         auto value = parse_internal(keep);
                         if (keep and not value.is_discarded())
                         {
                             result.push_back(std::move(value));
                         }
-                    }
-                    while (last_token == lexer::token_type::value_separator);
 
-                    // closing ]
-                    expect(lexer::token_type::end_array);
-                    get_token();
+                        // comma -> next value
+                        get_token();
+                        if (last_token == lexer::token_type::value_separator)
+                        {
+                            get_token();
+                            continue;
+                        }
+
+                        // closing ]
+                        expect(lexer::token_type::end_array);
+                        break;
+                    }
+
                     if (keep and callback and not callback(--depth, parse_event_t::array_end, result))
                     {
                         result = basic_json(value_t::discarded);
@@ -11536,22 +12828,18 @@ basic_json_parser_74:
 
                 case lexer::token_type::literal_null:
                 {
-                    get_token();
                     result.m_type = value_t::null;
                     break;
                 }
 
                 case lexer::token_type::value_string:
                 {
-                    const auto s = m_lexer.get_string();
-                    get_token();
-                    result = basic_json(s);
+                    result = basic_json(m_lexer.get_string());
                     break;
                 }
 
                 case lexer::token_type::literal_true:
                 {
-                    get_token();
                     result.m_type = value_t::boolean;
                     result.m_value = true;
                     break;
@@ -11559,25 +12847,49 @@ basic_json_parser_74:
 
                 case lexer::token_type::literal_false:
                 {
-                    get_token();
                     result.m_type = value_t::boolean;
                     result.m_value = false;
                     break;
                 }
 
                 case lexer::token_type::value_unsigned:
+                {
+                    result.m_type = value_t::number_unsigned;
+                    result.m_value = m_lexer.get_number_unsigned();
+                    break;
+                }
+
                 case lexer::token_type::value_integer:
+                {
+                    result.m_type = value_t::number_integer;
+                    result.m_value = m_lexer.get_number_integer();
+                    break;
+                }
+
                 case lexer::token_type::value_float:
                 {
-                    m_lexer.get_number(result, last_token);
-                    get_token();
+                    result.m_type = value_t::number_float;
+                    result.m_value = m_lexer.get_number_float();
+
+                    // throw in case of infinity or NAN
+                    if (JSON_UNLIKELY(not std::isfinite(result.m_value.number_float)))
+                    {
+                        JSON_THROW(out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'"));
+                    }
+
                     break;
                 }
 
+                case lexer::token_type::parse_error:
+                {
+                    // using "uninitialized" to avoid "expected" message
+                    expect(lexer::token_type::uninitialized);
+                }
+
                 default:
                 {
-                    // the last token was unexpected
-                    unexpect(last_token);
+                    // the last token was unexpected; we expected a value
+                    expect(lexer::token_type::literal_or_value);
                 }
             }
 
@@ -11588,36 +12900,167 @@ basic_json_parser_74:
             return result;
         }
 
+        /*!
+        @brief the acutal acceptor
+
+        @invariant 1. The last token is not yet processed. Therefore, the
+                      caller of this function must make sure a token has
+                      been read.
+                   2. When this function returns, the last token is processed.
+                      That is, the last read character was already considered.
+
+        This invariant makes sure that no token needs to be "unput".
+        */
+        bool accept_internal()
+        {
+            switch (last_token)
+            {
+                case lexer::token_type::begin_object:
+                {
+                    // read next token
+                    get_token();
+
+                    // closing } -> we are done
+                    if (last_token == lexer::token_type::end_object)
+                    {
+                        return true;
+                    }
+
+                    // parse values
+                    while (true)
+                    {
+                        // parse key
+                        if (last_token != lexer::token_type::value_string)
+                        {
+                            return false;
+                        }
+
+                        // parse separator (:)
+                        get_token();
+                        if (last_token != lexer::token_type::name_separator)
+                        {
+                            return false;
+                        }
+
+                        // parse value
+                        get_token();
+                        if (not accept_internal())
+                        {
+                            return false;
+                        }
+
+                        // comma -> next value
+                        get_token();
+                        if (last_token == lexer::token_type::value_separator)
+                        {
+                            get_token();
+                            continue;
+                        }
+
+                        // closing }
+                        if (last_token != lexer::token_type::end_object)
+                        {
+                            return false;
+                        }
+
+                        return true;
+                    }
+                }
+
+                case lexer::token_type::begin_array:
+                {
+                    // read next token
+                    get_token();
+
+                    // closing ] -> we are done
+                    if (last_token == lexer::token_type::end_array)
+                    {
+                        return true;
+                    }
+
+                    // parse values
+                    while (true)
+                    {
+                        // parse value
+                        if (not accept_internal())
+                        {
+                            return false;
+                        }
+
+                        // comma -> next value
+                        get_token();
+                        if (last_token == lexer::token_type::value_separator)
+                        {
+                            get_token();
+                            continue;
+                        }
+
+                        // closing ]
+                        if (last_token != lexer::token_type::end_array)
+                        {
+                            return false;
+                        }
+
+                        return true;
+                    }
+                }
+
+                case lexer::token_type::literal_false:
+                case lexer::token_type::literal_null:
+                case lexer::token_type::literal_true:
+                case lexer::token_type::value_float:
+                case lexer::token_type::value_integer:
+                case lexer::token_type::value_string:
+                case lexer::token_type::value_unsigned:
+                {
+                    return true;
+                }
+
+                default:
+                {
+                    // the last token was unexpected
+                    return false;
+                }
+            }
+        }
+
         /// get next token from lexer
         typename lexer::token_type get_token()
         {
-            last_token = m_lexer.scan();
-            return last_token;
+            return (last_token = m_lexer.scan());
         }
 
-        void expect(typename lexer::token_type t) const
+        /*!
+        @throw parse_error.101 if expected token did not occur
+        */
+        void expect(typename lexer::token_type t)
         {
-            if (t != last_token)
+            if (JSON_UNLIKELY(t != last_token))
             {
-                std::string error_msg = "parse error - unexpected ";
-                error_msg += (last_token == lexer::token_type::parse_error ? ("'" +  m_lexer.get_token_string() +
-                              "'") :
-                              lexer::token_type_name(last_token));
-                error_msg += "; expected " + lexer::token_type_name(t);
-                JSON_THROW(std::invalid_argument(error_msg));
+                errored = true;
+                expected = t;
+                throw_exception();
             }
         }
 
-        void unexpect(typename lexer::token_type t) const
+        [[noreturn]] void throw_exception() const
         {
-            if (t == last_token)
+            std::string error_msg = "syntax error - ";
+            if (last_token == lexer::token_type::parse_error)
             {
-                std::string error_msg = "parse error - unexpected ";
-                error_msg += (last_token == lexer::token_type::parse_error ? ("'" +  m_lexer.get_token_string() +
-                              "'") :
-                              lexer::token_type_name(last_token));
-                JSON_THROW(std::invalid_argument(error_msg));
+                error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" + m_lexer.get_token_string() + "'";
             }
+            else
+            {
+                error_msg += "unexpected " + std::string(lexer::token_type_name(last_token));
+            }
+
+            if (expected != lexer::token_type::uninitialized)
+            {
+                error_msg += "; expected " + std::string(lexer::token_type_name(expected));
+            }
+
+            JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg));
         }
 
       private:
@@ -11629,6 +13072,10 @@ basic_json_parser_74:
         typename lexer::token_type last_token = lexer::token_type::uninitialized;
         /// the lexer
         lexer m_lexer;
+        /// whether a syntax error occurred
+        bool errored = false;
+        /// possible reason for the syntax error
+        typename lexer::token_type expected = lexer::token_type::uninitialized;
     };
 
   public:
@@ -11659,12 +13106,12 @@ basic_json_parser_74:
                       empty string is assumed which references the whole JSON
                       value
 
-        @throw std::domain_error if reference token is nonempty and does not
-        begin with a slash (`/`); example: `"JSON pointer must be empty or
-        begin with /"`
-        @throw std::domain_error if a tilde (`~`) is not followed by `0`
-        (representing `~`) or `1` (representing `/`); example: `"escape error:
-        ~ must be followed with 0 or 1"`
+        @throw parse_error.107 if the given JSON pointer @a s is nonempty and
+        does not begin with a slash (`/`); see example below
+
+        @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s
+        is not followed by `0` (representing `~`) or `1` (representing `/`);
+        see example below
 
         @liveexample{The example shows the construction several valid JSON
         pointers as well as the exceptional behavior.,json_pointer}
@@ -11707,12 +13154,15 @@ basic_json_parser_74:
         }
 
       private:
-        /// remove and return last reference pointer
+        /*!
+        @brief remove and return last reference pointer
+        @throw out_of_range.405 if JSON pointer has no parent
+        */
         std::string pop_back()
         {
             if (is_root())
             {
-                JSON_THROW(std::domain_error("JSON pointer has no parent"));
+                JSON_THROW(out_of_range::create(405, "JSON pointer has no parent"));
             }
 
             auto last = reference_tokens.back();
@@ -11730,7 +13180,7 @@ basic_json_parser_74:
         {
             if (is_root())
             {
-                JSON_THROW(std::domain_error("JSON pointer has no parent"));
+                JSON_THROW(out_of_range::create(405, "JSON pointer has no parent"));
             }
 
             json_pointer result = *this;
@@ -11742,6 +13192,9 @@ basic_json_parser_74:
         @brief create and return a reference to the pointed to value
 
         @complexity Linear in the number of reference tokens.
+
+        @throw parse_error.109 if array index is not a number
+        @throw type_error.313 if value cannot be unflattened
         */
         reference get_and_create(reference j) const
         {
@@ -11778,7 +13231,14 @@ basic_json_parser_74:
                     case value_t::array:
                     {
                         // create an entry in the array
-                        result = &result->operator[](static_cast<size_type>(std::stoi(reference_token)));
+                        JSON_TRY
+                        {
+                            result = &result->operator[](static_cast<size_type>(std::stoi(reference_token)));
+                        }
+                        JSON_CATCH (std::invalid_argument&)
+                        {
+                            JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                        }
                         break;
                     }
 
@@ -11791,7 +13251,7 @@ basic_json_parser_74:
                     */
                     default:
                     {
-                        JSON_THROW(std::domain_error("invalid value to unflatten"));
+                        JSON_THROW(type_error::create(313, "invalid value to unflatten"));
                     }
                 }
             }
@@ -11814,9 +13274,9 @@ basic_json_parser_74:
 
         @complexity Linear in the length of the JSON pointer.
 
-        @throw std::out_of_range      if the JSON pointer can not be resolved
-        @throw std::domain_error      if an array index begins with '0'
-        @throw std::invalid_argument  if an array index was not a number
+        @throw parse_error.106   if an array index begins with '0'
+        @throw parse_error.109   if an array index was not a number
+        @throw out_of_range.404  if the JSON pointer can not be resolved
         */
         reference get_unchecked(pointer ptr) const
         {
@@ -11830,7 +13290,7 @@ basic_json_parser_74:
                                                   reference_token.end(),
                                                   [](const char x)
                     {
-                        return std::isdigit(x);
+                        return (x >= '0' and x <= '9');
                     });
 
                     // change value to array for numbers or "-" or to object
@@ -11859,7 +13319,7 @@ basic_json_parser_74:
                         // error condition (cf. RFC 6901, Sect. 4)
                         if (reference_token.size() > 1 and reference_token[0] == '0')
                         {
-                            JSON_THROW(std::domain_error("array index must not begin with '0'"));
+                            JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'"));
                         }
 
                         if (reference_token == "-")
@@ -11870,14 +13330,21 @@ basic_json_parser_74:
                         else
                         {
                             // convert array index to number; unchecked access
-                            ptr = &ptr->operator[](static_cast<size_type>(std::stoi(reference_token)));
+                            JSON_TRY
+                            {
+                                ptr = &ptr->operator[](static_cast<size_type>(std::stoi(reference_token)));
+                            }
+                            JSON_CATCH (std::invalid_argument&)
+                            {
+                                JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                            }
                         }
                         break;
                     }
 
                     default:
                     {
-                        JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'"));
+                        JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
                     }
                 }
             }
@@ -11885,6 +13352,12 @@ basic_json_parser_74:
             return *ptr;
         }
 
+        /*!
+        @throw parse_error.106   if an array index begins with '0'
+        @throw parse_error.109   if an array index was not a number
+        @throw out_of_range.402  if the array index '-' is used
+        @throw out_of_range.404  if the JSON pointer can not be resolved
+        */
         reference get_checked(pointer ptr) const
         {
             for (const auto& reference_token : reference_tokens)
@@ -11903,25 +13376,32 @@ basic_json_parser_74:
                         if (reference_token == "-")
                         {
                             // "-" always fails the range check
-                            JSON_THROW(std::out_of_range("array index '-' (" +
-                                                         std::to_string(ptr->m_value.array->size()) +
-                                                         ") is out of range"));
+                            JSON_THROW(out_of_range::create(402, "array index '-' (" +
+                                                            std::to_string(ptr->m_value.array->size()) +
+                                                            ") is out of range"));
                         }
 
                         // error condition (cf. RFC 6901, Sect. 4)
                         if (reference_token.size() > 1 and reference_token[0] == '0')
                         {
-                            JSON_THROW(std::domain_error("array index must not begin with '0'"));
+                            JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'"));
                         }
 
                         // note: at performs range check
-                        ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                        JSON_TRY
+                        {
+                            ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                        }
+                        JSON_CATCH (std::invalid_argument&)
+                        {
+                            JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                        }
                         break;
                     }
 
                     default:
                     {
-                        JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'"));
+                        JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
                     }
                 }
             }
@@ -11936,6 +13416,11 @@ basic_json_parser_74:
 
         @return const reference to the JSON value pointed to by the JSON
                 pointer
+
+        @throw parse_error.106   if an array index begins with '0'
+        @throw parse_error.109   if an array index was not a number
+        @throw out_of_range.402  if the array index '-' is used
+        @throw out_of_range.404  if the JSON pointer can not be resolved
         */
         const_reference get_unchecked(const_pointer ptr) const
         {
@@ -11955,25 +13440,32 @@ basic_json_parser_74:
                         if (reference_token == "-")
                         {
                             // "-" cannot be used for const access
-                            JSON_THROW(std::out_of_range("array index '-' (" +
-                                                         std::to_string(ptr->m_value.array->size()) +
-                                                         ") is out of range"));
+                            JSON_THROW(out_of_range::create(402, "array index '-' (" +
+                                                            std::to_string(ptr->m_value.array->size()) +
+                                                            ") is out of range"));
                         }
 
                         // error condition (cf. RFC 6901, Sect. 4)
                         if (reference_token.size() > 1 and reference_token[0] == '0')
                         {
-                            JSON_THROW(std::domain_error("array index must not begin with '0'"));
+                            JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'"));
                         }
 
                         // use unchecked array access
-                        ptr = &ptr->operator[](static_cast<size_type>(std::stoi(reference_token)));
+                        JSON_TRY
+                        {
+                            ptr = &ptr->operator[](static_cast<size_type>(std::stoi(reference_token)));
+                        }
+                        JSON_CATCH (std::invalid_argument&)
+                        {
+                            JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                        }
                         break;
                     }
 
                     default:
                     {
-                        JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'"));
+                        JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
                     }
                 }
             }
@@ -11981,6 +13473,12 @@ basic_json_parser_74:
             return *ptr;
         }
 
+        /*!
+        @throw parse_error.106   if an array index begins with '0'
+        @throw parse_error.109   if an array index was not a number
+        @throw out_of_range.402  if the array index '-' is used
+        @throw out_of_range.404  if the JSON pointer can not be resolved
+        */
         const_reference get_checked(const_pointer ptr) const
         {
             for (const auto& reference_token : reference_tokens)
@@ -11999,25 +13497,32 @@ basic_json_parser_74:
                         if (reference_token == "-")
                         {
                             // "-" always fails the range check
-                            JSON_THROW(std::out_of_range("array index '-' (" +
-                                                         std::to_string(ptr->m_value.array->size()) +
-                                                         ") is out of range"));
+                            JSON_THROW(out_of_range::create(402, "array index '-' (" +
+                                                            std::to_string(ptr->m_value.array->size()) +
+                                                            ") is out of range"));
                         }
 
                         // error condition (cf. RFC 6901, Sect. 4)
                         if (reference_token.size() > 1 and reference_token[0] == '0')
                         {
-                            JSON_THROW(std::domain_error("array index must not begin with '0'"));
+                            JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'"));
                         }
 
                         // note: at performs range check
-                        ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                        JSON_TRY
+                        {
+                            ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                        }
+                        JSON_CATCH (std::invalid_argument&)
+                        {
+                            JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                        }
                         break;
                     }
 
                     default:
                     {
-                        JSON_THROW(std::out_of_range("unresolved reference token '" + reference_token + "'"));
+                        JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
                     }
                 }
             }
@@ -12025,7 +13530,15 @@ basic_json_parser_74:
             return *ptr;
         }
 
-        /// split the string input to reference tokens
+        /*!
+        @brief split the string input to reference tokens
+
+        @note This function is only called by the json_pointer constructor.
+              All exceptions below are documented there.
+
+        @throw parse_error.107  if the pointer is not empty or begins with '/'
+        @throw parse_error.108  if character '~' is not followed by '0' or '1'
+        */
         static std::vector<std::string> split(const std::string& reference_string)
         {
             std::vector<std::string> result;
@@ -12039,7 +13552,7 @@ basic_json_parser_74:
             // check if nonempty reference string begins with slash
             if (reference_string[0] != '/')
             {
-                JSON_THROW(std::domain_error("JSON pointer must be empty or begin with '/'"));
+                JSON_THROW(parse_error::create(107, 1, "JSON pointer must be empty or begin with '/' - was: '" + reference_string + "'"));
             }
 
             // extract the reference tokens:
@@ -12074,7 +13587,7 @@ basic_json_parser_74:
                             (reference_token[pos + 1] != '0' and
                              reference_token[pos + 1] != '1'))
                     {
-                        JSON_THROW(std::domain_error("escape error: '~' must be followed with '0' or '1'"));
+                        JSON_THROW(parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'"));
                     }
                 }
 
@@ -12086,7 +13599,6 @@ basic_json_parser_74:
             return result;
         }
 
-      private:
         /*!
         @brief replace all occurrences of a substring by another string
 
@@ -12095,7 +13607,8 @@ basic_json_parser_74:
         @param[in]     f  the substring to replace with @a t
         @param[in]     t  the string to replace @a f
 
-        @pre The search string @a f must not be empty.
+        @pre The search string @a f must not be empty. **This precondition is
+             enforced with an assertion.**
 
         @since version 2.0.0
         */
@@ -12195,12 +13708,17 @@ basic_json_parser_74:
         @param[in] value  flattened JSON
 
         @return unflattened JSON
+
+        @throw parse_error.109 if array index is not a number
+        @throw type_error.314  if value is not an object
+        @throw type_error.315  if object values are not primitive
+        @throw type_error.313  if value cannot be unflattened
         */
         static basic_json unflatten(const basic_json& value)
         {
             if (not value.is_object())
             {
-                JSON_THROW(std::domain_error("only objects can be unflattened"));
+                JSON_THROW(type_error::create(314, "only objects can be unflattened"));
             }
 
             basic_json result;
@@ -12210,7 +13728,7 @@ basic_json_parser_74:
             {
                 if (not element.second.is_primitive())
                 {
-                    JSON_THROW(std::domain_error("values in object must be primitive"));
+                    JSON_THROW(type_error::create(315, "values in object must be primitive"));
                 }
 
                 // assign value to reference pointed to by JSON pointer; Note
@@ -12224,7 +13742,6 @@ basic_json_parser_74:
             return result;
         }
 
-      private:
         friend bool operator==(json_pointer const& lhs,
                                json_pointer const& rhs) noexcept
         {
@@ -12273,9 +13790,9 @@ basic_json_parser_74:
 
     @complexity Constant.
 
-    @throw std::out_of_range      if the JSON pointer can not be resolved
-    @throw std::domain_error      if an array index begins with '0'
-    @throw std::invalid_argument  if an array index was not a number
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.404  if the JSON pointer can not be resolved
 
     @liveexample{The behavior is shown in the example.,operatorjson_pointer}
 
@@ -12300,9 +13817,10 @@ basic_json_parser_74:
 
     @complexity Constant.
 
-    @throw std::out_of_range      if the JSON pointer can not be resolved
-    @throw std::domain_error      if an array index begins with '0'
-    @throw std::invalid_argument  if an array index was not a number
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
 
     @liveexample{The behavior is shown in the example.,operatorjson_pointer_const}
 
@@ -12323,15 +13841,30 @@ basic_json_parser_74:
 
     @return reference to the element pointed to by @a ptr
 
+    @throw parse_error.106 if an array index in the passed JSON pointer @a ptr
+    begins with '0'. See example below.
+
+    @throw parse_error.109 if an array index in the passed JSON pointer @a ptr
+    is not a number. See example below.
+
+    @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr
+    is out of range. See example below.
+
+    @throw out_of_range.402 if the array index '-' is used in the passed JSON
+    pointer @a ptr. As `at` provides checked access (and no elements are
+    implicitly inserted), the index '-' is always invalid. See example below.
+
+    @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
+    See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
     @complexity Constant.
 
-    @throw std::out_of_range      if the JSON pointer can not be resolved
-    @throw std::domain_error      if an array index begins with '0'
-    @throw std::invalid_argument  if an array index was not a number
+    @since version 2.0.0
 
     @liveexample{The behavior is shown in the example.,at_json_pointer}
-
-    @since version 2.0.0
     */
     reference at(const json_pointer& ptr)
     {
@@ -12348,15 +13881,30 @@ basic_json_parser_74:
 
     @return reference to the element pointed to by @a ptr
 
+    @throw parse_error.106 if an array index in the passed JSON pointer @a ptr
+    begins with '0'. See example below.
+
+    @throw parse_error.109 if an array index in the passed JSON pointer @a ptr
+    is not a number. See example below.
+
+    @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr
+    is out of range. See example below.
+
+    @throw out_of_range.402 if the array index '-' is used in the passed JSON
+    pointer @a ptr. As `at` provides checked access (and no elements are
+    implicitly inserted), the index '-' is always invalid. See example below.
+
+    @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved.
+    See example below.
+
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
     @complexity Constant.
 
-    @throw std::out_of_range      if the JSON pointer can not be resolved
-    @throw std::domain_error      if an array index begins with '0'
-    @throw std::invalid_argument  if an array index was not a number
+    @since version 2.0.0
 
     @liveexample{The behavior is shown in the example.,at_json_pointer_const}
-
-    @since version 2.0.0
     */
     const_reference at(const json_pointer& ptr) const
     {
@@ -12412,6 +13960,9 @@ basic_json_parser_74:
 
     @complexity Linear in the size the JSON value.
 
+    @throw type_error.314  if value is not an object
+    @throw type_error.315  if object values are not primitve
+
     @liveexample{The following code shows how a flattened JSON object is
     unflattened into the original nested JSON object.,unflatten}
 
@@ -12449,12 +14000,23 @@ basic_json_parser_74:
           any case, the original value is not changed: the patch is applied
           to a copy of the value.
 
-    @throw std::out_of_range if a JSON pointer inside the patch could not
-    be resolved successfully in the current JSON value; example: `"key baz
-    not found"`
-    @throw invalid_argument if the JSON patch is malformed (e.g., mandatory
+    @throw parse_error.104 if the JSON patch does not consist of an array of
+    objects
+
+    @throw parse_error.105 if the JSON patch is malformed (e.g., mandatory
     attributes are missing); example: `"operation add must have member path"`
 
+    @throw out_of_range.401 if an array index is out of range.
+
+    @throw out_of_range.403 if a JSON pointer inside the patch could not be
+    resolved successfully in the current JSON value; example: `"key baz not
+    found"`
+
+    @throw out_of_range.405 if JSON pointer has no parent ("add", "remove",
+    "move")
+
+    @throw other_error.501 if "test" operation was unsuccessful
+
     @complexity Linear in the size of the JSON value and the length of the
     JSON patch. As usually only a fraction of the JSON value is affected by
     the patch, the complexity can usually be neglected.
@@ -12477,7 +14039,7 @@ basic_json_parser_74:
         // the valid JSON Patch operations
         enum class patch_operations {add, remove, replace, move, copy, test, invalid};
 
-        const auto get_op = [](const std::string op)
+        const auto get_op = [](const std::string & op)
         {
             if (op == "add")
             {
@@ -12551,7 +14113,7 @@ basic_json_parser_74:
                             if (static_cast<size_type>(idx) > parent.size())
                             {
                                 // avoid undefined behavior
-                                JSON_THROW(std::out_of_range("array index " + std::to_string(idx) + " is out of range"));
+                                JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
                             }
                             else
                             {
@@ -12589,7 +14151,7 @@ basic_json_parser_74:
                 }
                 else
                 {
-                    JSON_THROW(std::out_of_range("key '" + last_path + "' not found"));
+                    JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found"));
                 }
             }
             else if (parent.is_array())
@@ -12599,11 +14161,10 @@ basic_json_parser_74:
             }
         };
 
-        // type check
+        // type check: top level value must be an array
         if (not json_patch.is_array())
         {
-            // a JSON patch must be an array of objects
-            JSON_THROW(std::invalid_argument("JSON patch must be an array of objects"));
+            JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
         }
 
         // iterate and apply the operations
@@ -12623,23 +14184,23 @@ basic_json_parser_74:
                 // check if desired value is present
                 if (it == val.m_value.object->end())
                 {
-                    JSON_THROW(std::invalid_argument(error_msg + " must have member '" + member + "'"));
+                    JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'"));
                 }
 
                 // check if result is of type string
                 if (string_type and not it->second.is_string())
                 {
-                    JSON_THROW(std::invalid_argument(error_msg + " must have string member '" + member + "'"));
+                    JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'"));
                 }
 
                 // no error: return value
                 return it->second;
             };
 
-            // type check
+            // type check: every element of the array must be an object
             if (not val.is_object())
             {
-                JSON_THROW(std::invalid_argument("JSON patch must be an array of objects"));
+                JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
             }
 
             // collect mandatory members
@@ -12687,7 +14248,7 @@ basic_json_parser_74:
 
                 case patch_operations::copy:
                 {
-                    const std::string from_path = get_value("copy", "from", true);;
+                    const std::string from_path = get_value("copy", "from", true);
                     const json_pointer from_ptr(from_path);
 
                     // the "from" location must exist - use at()
@@ -12704,7 +14265,7 @@ basic_json_parser_74:
                         // the "path" location must exist - use at()
                         success = (result.at(ptr) == get_value("test", "value", false));
                     }
-                    JSON_CATCH (std::out_of_range&)
+                    JSON_CATCH (out_of_range&)
                     {
                         // ignore out of range errors: success remains false
                     }
@@ -12712,7 +14273,7 @@ basic_json_parser_74:
                     // throw an exception if test fails
                     if (not success)
                     {
-                        JSON_THROW(std::domain_error("unsuccessful: " + val.dump()));
+                        JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump()));
                     }
 
                     break;
@@ -12722,7 +14283,7 @@ basic_json_parser_74:
                 {
                     // op must be "add", "remove", "replace", "move", "copy", or
                     // "test"
-                    JSON_THROW(std::invalid_argument("operation value '" + op + "' is invalid"));
+                    JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid"));
                 }
             }
         }
@@ -12951,6 +14512,22 @@ struct hash<nlohmann::json>
         return h(j.dump());
     }
 };
+
+/// specialization for std::less<value_t>
+template <>
+struct less<::nlohmann::detail::value_t>
+{
+    /*!
+    @brief compare two value_t enum values
+    @since version 3.0.0
+    */
+    bool operator()(nlohmann::detail::value_t lhs,
+                    nlohmann::detail::value_t rhs) const noexcept
+    {
+        return nlohmann::detail::operator<(lhs, rhs);
+    }
+};
+
 } // namespace std
 
 /*!
@@ -12993,11 +14570,16 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std
 #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__)
     #pragma GCC diagnostic pop
 #endif
+#if defined(__clang__)
+    #pragma GCC diagnostic pop
+#endif
 
 // clean up
 #undef JSON_CATCH
-#undef JSON_DEPRECATED
 #undef JSON_THROW
 #undef JSON_TRY
+#undef JSON_LIKELY
+#undef JSON_UNLIKELY
+#undef JSON_DEPRECATED
 
 #endif
diff --git a/lib/serialisation/BaseIO.h b/lib/serialisation/BaseIO.h
index ec3eefa0..24e1cec7 100644
--- a/lib/serialisation/BaseIO.h
+++ b/lib/serialisation/BaseIO.h
@@ -86,7 +86,7 @@ namespace Grid {
                                       or element<T>::is_number;
   };
   
-  // Vector flatening utility class ////////////////////////////////////////////
+  // Vector flattening utility class ////////////////////////////////////////////
   // Class to flatten a multidimensional std::vector
   template <typename V>
   class Flatten
diff --git a/lib/serialisation/JSON_IO.cc b/lib/serialisation/JSON_IO.cc
index 0a3968e9..ae37a4e5 100644
--- a/lib/serialisation/JSON_IO.cc
+++ b/lib/serialisation/JSON_IO.cc
@@ -42,6 +42,7 @@ JSONWriter::~JSONWriter(void)
 
   // write prettified JSON to file
   std::ofstream os(fileName_);
+  std::cout << "JSONWriter::~JSONWriter" << std::endl;
   os << std::setw(2) << json::parse(ss_.str()) << std::endl;
 }
 
@@ -56,6 +57,7 @@ void JSONWriter::push(const string &s)
 
 void JSONWriter::pop(void)
 {
+  std::cout << "JSONWriter::pop" << std::endl;
   delete_comma();
   ss_ << "},";
 }
diff --git a/lib/serialisation/JSON_IO.h b/lib/serialisation/JSON_IO.h
index fc5e9631..9c247d27 100644
--- a/lib/serialisation/JSON_IO.h
+++ b/lib/serialisation/JSON_IO.h
@@ -106,7 +106,7 @@ namespace Grid
   template <typename U>
   void JSONWriter::writeDefault(const std::string &s, const U &x)
   {
-    //std::cout << "JSONReader::writeDefault(U) : " << s <<  std::endl;
+    //std::cout << "JSONWriter::writeDefault(U) : " << s <<  std::endl;
     std::ostringstream os;
     os << std::boolalpha << x;
     if (s.size())
@@ -115,10 +115,25 @@ namespace Grid
      ss_ << os.str() << " ," ;
   }
 
+  // specialize for string
+  template <>
+  void JSONWriter::writeDefault(const std::string &s, const std::string &x)
+  {
+    //std::cout << "JSONWriter::writeDefault(U) : " << s <<  std::endl;
+    std::ostringstream os;
+    os << std::boolalpha << x;
+    if (s.size())
+      ss_ << "\""<< s << "\" : \"" << os.str() << "\" ," ;
+    else
+     ss_ << os.str() << " ," ;
+  }
+
+
+
   template <typename U>
   void JSONWriter::writeDefault(const std::string &s, const std::complex<U> &x)
   {
-    //std::cout << "JSONReader::writeDefault(complex) : " << s <<  std::endl;
+    //std::cout << "JSONWriter::writeDefault(complex) : " << s <<  " " << x <<  std::endl;
     std::ostringstream os;
     os << "["<< std::boolalpha << x.real() << ", " << x.imag() << "]";
     if (s.size())
@@ -130,7 +145,7 @@ namespace Grid
   template <typename U>
   void JSONWriter::writeDefault(const std::string &s, const std::vector<U> &x)
   {
-    //std::cout << "JSONReader::writeDefault(vec U) : " << s <<  std::endl;
+    //std::cout << "JSONWriter::writeDefault(vec U) : " << s <<  std::endl;
 
     if (s.size())
       ss_ << " \""<<s<<"\" : [";
@@ -146,12 +161,12 @@ namespace Grid
 
   template<std::size_t N>
   void JSONWriter::writeDefault(const std::string &s, const char(&x)[N]){
-    //std::cout << "JSONReader::writeDefault(char U) : " << s <<  std::endl;
+    //std::cout << "JSONWriter::writeDefault(char U) : " << s <<  "  " << x << std::endl;
 
     if (s.size())
-    ss_ << "\""<< s << "\" : \"" << x << "\" ," ;
+      ss_ << "\""<< s << "\" : \"" << x << "\" ," ;
     else
-    ss_ << "\"" << x << "\" ," ;
+      ss_ << "\"" << x << "\" ," ;
   }
 
   // Reader template implementation ////////////////////////////////////////////
@@ -177,7 +192,7 @@ namespace Grid
   void JSONReader::readDefault(const std::string &s, std::complex<U> &output)
   {
     U tmp1, tmp2;
-    //std::cout << "JSONReader::readDefault( complex U) : " << s << "  :  "<< jcur_ << std::endl;
+    //std::cout << "JSONReader::readDefault(complex U) : " << s << "  :  "<< jcur_ << std::endl;
     json j = jcur_;
     json::iterator it = j.begin();
     jcur_ = *it;
diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc
index 3988784c..8462fc5d 100644
--- a/tests/IO/Test_serialisation.cc
+++ b/tests/IO/Test_serialisation.cc
@@ -1,6 +1,6 @@
     /*************************************************************************************
 
-    Grid physics library, www.github.com/paboyle/Grid 
+    Grid physics library, www.github.com/paboyle/Grid
 
     Source file: ./tests/Test_serialisation.cc
 
@@ -34,7 +34,7 @@ using namespace Grid;
 using namespace Grid::QCD;
 
 GRID_SERIALIZABLE_ENUM(myenum, undef, red, 1, blue, 2, green, 3);
-  
+
 class myclass: Serializable {
 public:
   GRID_SERIALIZABLE_CLASS_MEMBERS(myclass,
@@ -79,14 +79,14 @@ void ioTest(const std::string &filename, const O &object, const std::string &nam
   // writer needs to be destroyed so that writing physically happens
   {
     W writer(filename);
-    
+
     write(writer, "testobject", object);
   }
-  
+
   R    reader(filename);
   O    buf;
   bool good;
-  
+
   read(reader, "testobject", buf);
   good = (object == buf);
   std::cout << name << " IO test: " << (good ? "success" : "failure");
@@ -98,7 +98,7 @@ int main(int argc,char **argv)
 {
   std::cout << "==== basic IO" << std::endl;
   XmlWriter WR("bother.xml");
-  
+
   // test basic type writing
   std::cout << "-- basic writing to 'bother.xml'..." << std::endl;
   push(WR,"BasicTypes");
@@ -112,12 +112,12 @@ int main(int argc,char **argv)
   write(WR,"d",d);
   write(WR,"b",b);
   pop(WR);
-  
+
   // test serializable class writing
   myclass              obj(1234); // non-trivial constructor
   std::vector<myclass> vec;
   std::pair<myenum, myenum> pair;
-  
+
   std::cout << "-- serialisable class writing to 'bother.xml'..." << std::endl;
   write(WR,"obj",obj);
   WR.write("obj2", obj);
@@ -132,11 +132,11 @@ int main(int argc,char **argv)
   std::cout << "-- serialisable class comparison:" << std::endl;
   std::cout << "vec[0] == obj: " << ((vec[0] == obj) ? "true" : "false") << std::endl;
   std::cout << "vec[1] == obj: " << ((vec[1] == obj) ? "true" : "false") << std::endl;
-  
+
   write(WR, "objpair", pair);
   std::cout << "-- pair writing to std::cout:" << std::endl;
   std::cout << pair << std::endl;
-  
+
   // read tests
   std::cout << "\n==== IO self-consistency tests" << std::endl;
   //// XML
@@ -154,7 +154,7 @@ int main(int argc,char **argv)
   //// text
   ioTest<JSONWriter, JSONReader>("iotest.json", obj, "JSON   (object)           ");
   ioTest<JSONWriter, JSONReader>("iotest.json", vec, "JSON   (vector of objects)");
-  ioTest<JSONWriter, JSONReader>("iotest.json", pair, "JSON   (pair of objects)");
+  //ioTest<JSONWriter, JSONReader>("iotest.json", pair, "JSON   (pair of objects)");
 
   //// HDF5
 #undef HAVE_HDF5
@@ -163,13 +163,13 @@ int main(int argc,char **argv)
   ioTest<Hdf5Writer, Hdf5Reader>("iotest.h5", vec, "HDF5   (vector of objects)");
   ioTest<Hdf5Writer, Hdf5Reader>("iotest.h5", pair, "HDF5   (pair of objects)");
 #endif
-  
+
   std::cout << "\n==== vector flattening/reconstruction" << std::endl;
   typedef std::vector<std::vector<std::vector<double>>> vec3d;
-  
+
   vec3d dv, buf;
   double d = 0.;
-  
+
   dv.resize(4);
   for (auto &v1: dv)
   {
@@ -185,66 +185,71 @@ int main(int argc,char **argv)
   }
   std::cout << "original 3D vector:" << std::endl;
   std::cout << dv << std::endl;
-  
+
   Flatten<vec3d> flatdv(dv);
-  
+
   std::cout << "\ndimensions:" << std::endl;
   std::cout << flatdv.getDim() << std::endl;
   std::cout << "\nflattened vector:" << std::endl;
   std::cout << flatdv.getFlatVector() << std::endl;
-  
+
   Reconstruct<vec3d> rec(flatdv.getFlatVector(), flatdv.getDim());
   std::cout << "\nreconstructed vector:" << std::endl;
   std::cout << flatdv.getVector() << std::endl;
   std::cout << std::endl;
 
 
-//  std::cout << ".:::::: Testing JSON classes "<< std::endl;
-//
-//
-//  {
-//    JSONWriter JW("bother.json");
-//    
-//    // test basic type writing
-//    push(JW,"BasicTypes");
-//    write(JW,std::string("i16"),i16);
-//    write(JW,"u16",u16);
-//    write(JW,"i32",i32);
-//    write(JW,"u32",u32);
-//    write(JW,"i64",i64);
-//    write(JW,"u64",u64);
-//    write(JW,"f",f);
-//    write(JW,"d",d);
-//    write(JW,"b",b);
-//    pop(JW);
-//    
-//    // test serializable class writing
-//    myclass obj(1234); // non-trivial constructor
-//    std::cout << "-- serialisable class writing to 'bother.json'..." << std::endl;
-//    write(JW,"obj",obj);
-//    JW.write("obj2", obj);
-//    
-//    std::cout << obj << std::endl;
-//    
-//    std::vector<myclass> vec;
-//    vec.push_back(myclass(1234));
-//    vec.push_back(myclass(5678));
-//    vec.push_back(myclass(3838));
-//    write(JW, "objvec", vec);
-//    
-//  }
-//
-//  {
-//    JSONReader RD("bother.json");
-//    myclass jcopy1;
-//    std::vector<myclass> jveccopy1;
-//    read(RD,"obj",jcopy1);
-//    read(RD,"objvec", jveccopy1);
-//    std::cout << "Loaded (JSON) -----------------" << std::endl;
-//    std::cout << jcopy1 << std::endl << jveccopy1 << std::endl;
-//  }
-  
-/* 
+  std::cout << ".:::::: Testing JSON classes "<< std::endl;
+
+
+  {
+    JSONWriter JW("bother.json");
+
+    // test basic type writing
+    myenum a = myenum::red;
+    push(JW,"BasicTypes");
+    write(JW,std::string("i16"),i16);
+    write(JW,"myenum",a);
+    write(JW,"u16",u16);
+    write(JW,"i32",i32);
+    write(JW,"u32",u32);
+    write(JW,"i64",i64);
+    write(JW,"u64",u64);
+    write(JW,"f",f);
+    write(JW,"d",d);
+    write(JW,"b",b);
+    pop(JW);
+
+
+    // test serializable class writing
+    myclass obj(1234); // non-trivial constructor
+    std::cout << obj << std::endl;
+    std::cout << "-- serialisable class writing to 'bother.json'..." << std::endl;
+    write(JW,"obj",obj);
+    JW.write("obj2", obj);
+
+
+    std::vector<myclass> vec;
+    vec.push_back(myclass(1234));
+    vec.push_back(myclass(5678));
+    vec.push_back(myclass(3838));
+    write(JW, "objvec", vec);
+
+  }
+
+
+  {
+    JSONReader RD("bother.json");
+    myclass jcopy1;
+    std::vector<myclass> jveccopy1;
+    read(RD,"obj",jcopy1);
+    read(RD,"objvec", jveccopy1);
+    std::cout << "Loaded (JSON) -----------------" << std::endl;
+    std::cout << jcopy1 << std::endl << jveccopy1 << std::endl;
+  }
+ 
+
+/*
   // This is still work in progress
   {
     // Testing the next element function

From 184af5bd05267bd54e3322a04ec431a84164b708 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Wed, 12 Jul 2017 14:44:53 +0100
Subject: [PATCH 021/115] Added support for std::pair in the JSON serialiser

---
 lib/serialisation/JSON_IO.cc   | 11 +++++----
 lib/serialisation/JSON_IO.h    | 45 ++++++++++++++++++++++++++++++++--
 tests/IO/Test_serialisation.cc |  7 +++---
 3 files changed, 52 insertions(+), 11 deletions(-)

diff --git a/lib/serialisation/JSON_IO.cc b/lib/serialisation/JSON_IO.cc
index ae37a4e5..303dfcb7 100644
--- a/lib/serialisation/JSON_IO.cc
+++ b/lib/serialisation/JSON_IO.cc
@@ -42,7 +42,7 @@ JSONWriter::~JSONWriter(void)
 
   // write prettified JSON to file
   std::ofstream os(fileName_);
-  std::cout << "JSONWriter::~JSONWriter" << std::endl;
+  //std::cout << "JSONWriter::~JSONWriter" << std::endl;
   os << std::setw(2) << json::parse(ss_.str()) << std::endl;
 }
 
@@ -57,7 +57,7 @@ void JSONWriter::push(const string &s)
 
 void JSONWriter::pop(void)
 {
-  std::cout << "JSONWriter::pop" << std::endl;
+  //std::cout << "JSONWriter::pop" << std::endl;
   delete_comma();
   ss_ << "},";
 }
@@ -69,6 +69,7 @@ void JSONWriter::delete_comma()
   ss_.str(dlast);
 }
 
+/*
 // here we are hitting a g++ bug (Bug 56480)
 // compiles fine with clang
 // have to wrap in the Grid namespace
@@ -76,8 +77,7 @@ void JSONWriter::delete_comma()
 namespace Grid
 {
   template<>
-  void JSONWriter::writeDefault(const std::string &s,
-				const std::string &x)
+  void JSONWriter::writeDefault(const std::string &s,	const std::string &x)
   {
     if (s.size())
       ss_ << "\""<< s << "\" : \"" << x << "\" ," ; 
@@ -85,7 +85,7 @@ namespace Grid
       ss_ << "\"" << x << "\" ," ;
   }
 }// namespace Grid 
-
+*/
 
 // Reader implementation ///////////////////////////////////////////////////////
 JSONReader::JSONReader(const string &fileName)
@@ -140,6 +140,7 @@ void JSONReader::pop(void)
 
 bool JSONReader::nextElement(const std::string &s)
 {
+  // Work in progress
   // JSON dictionaries do not support multiple names 
   // Same name objects must be packed in vectors
   ++it_;
diff --git a/lib/serialisation/JSON_IO.h b/lib/serialisation/JSON_IO.h
index 9c247d27..a6d54fb3 100644
--- a/lib/serialisation/JSON_IO.h
+++ b/lib/serialisation/JSON_IO.h
@@ -58,10 +58,13 @@ namespace Grid
     void writeDefault(const std::string &s, const std::complex<U> &x);
     template <typename U>
     void writeDefault(const std::string &s, const std::vector<U> &x);
+    template <typename U, typename P>
+    void writeDefault(const std::string &s, const std::pair<U,P> &x);
 
     template<std::size_t N>
     void writeDefault(const std::string &s, const char(&x)[N]);
 
+
   private:
     void delete_comma();
     std::string         fileName_;
@@ -82,6 +85,8 @@ namespace Grid
     void readDefault(const std::string &s, std::complex<U> &output);
     template <typename U>
     void readDefault(const std::string &s, std::vector<U> &output);
+    template <typename U, typename P>
+    void readDefault(const std::string &s, std::pair<U,P> &output);
   private:
     json                jobject_; // main object
     json                jcur_;  // current json object
@@ -106,7 +111,7 @@ namespace Grid
   template <typename U>
   void JSONWriter::writeDefault(const std::string &s, const U &x)
   {
-    //std::cout << "JSONWriter::writeDefault(U) : " << s <<  std::endl;
+    //std::cout << "JSONWriter::writeDefault(U) : " << s <<  " " << x <<std::endl;
     std::ostringstream os;
     os << std::boolalpha << x;
     if (s.size())
@@ -119,7 +124,7 @@ namespace Grid
   template <>
   void JSONWriter::writeDefault(const std::string &s, const std::string &x)
   {
-    //std::cout << "JSONWriter::writeDefault(U) : " << s <<  std::endl;
+    //std::cout << "JSONWriter::writeDefault(string) : " << s <<  std::endl;
     std::ostringstream os;
     os << std::boolalpha << x;
     if (s.size())
@@ -142,6 +147,18 @@ namespace Grid
      ss_ << os.str() << " ," ;
   }
 
+  template <typename U, typename P>
+  void JSONWriter::writeDefault(const std::string &s, const std::pair<U,P> &x)
+  {
+    //std::cout << "JSONWriter::writeDefault(pair) : " << s <<  " " << x <<  std::endl;
+    std::ostringstream os;
+    os << "["<< std::boolalpha << "\""<< x.first << "\" , \"" << x.second << "\" ]";
+    if (s.size())
+      ss_ << "\""<< s << "\" : " << os.str() << " ," ;
+    else
+     ss_ << os.str() << " ," ;
+  }
+
   template <typename U>
   void JSONWriter::writeDefault(const std::string &s, const std::vector<U> &x)
   {
@@ -188,6 +205,30 @@ namespace Grid
 
   }
 
+  // Reader template implementation ////////////////////////////////////////////
+  template <typename U, typename P>
+  void JSONReader::readDefault(const std::string &s, std::pair<U,P> &output)
+  {
+    U first;
+    P second;
+    json j;
+    if (s.size()){
+      //std::cout << "JSONReader::readDefault(pair) : " << s << "  |  "<< jcur_[s] << std::endl;
+      j = jcur_[s];
+    } else {
+      j = jcur_;
+    }
+    json::iterator it = j.begin();
+    jcur_ = *it;
+    read("", first);
+    it++;
+    jcur_ = *it;
+    read("", second);
+    output = std::pair<U,P>(first,second);
+  }
+
+
+
   template <typename U>
   void JSONReader::readDefault(const std::string &s, std::complex<U> &output)
   {
diff --git a/tests/IO/Test_serialisation.cc b/tests/IO/Test_serialisation.cc
index 8462fc5d..82638ad9 100644
--- a/tests/IO/Test_serialisation.cc
+++ b/tests/IO/Test_serialisation.cc
@@ -29,7 +29,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 #include <Grid/Grid.h>
 
-
 using namespace Grid;
 using namespace Grid::QCD;
 
@@ -152,9 +151,9 @@ int main(int argc,char **argv)
   ioTest<TextWriter, TextReader>("iotest.dat", vec, "text   (vector of objects)");
   ioTest<TextWriter, TextReader>("iotest.dat", pair, "text   (pair of objects)");
   //// text
-  ioTest<JSONWriter, JSONReader>("iotest.json", obj, "JSON   (object)           ");
-  ioTest<JSONWriter, JSONReader>("iotest.json", vec, "JSON   (vector of objects)");
-  //ioTest<JSONWriter, JSONReader>("iotest.json", pair, "JSON   (pair of objects)");
+  ioTest<JSONWriter, JSONReader>("iotest.json", obj,  "JSON   (object)           ");
+  ioTest<JSONWriter, JSONReader>("iotest.json", vec,  "JSON   (vector of objects)");
+  ioTest<JSONWriter, JSONReader>("iotest.json", pair, "JSON   (pair of objects)");
 
   //// HDF5
 #undef HAVE_HDF5

From f64fb7bd77a775e4f9cde55a19cd6a8446af5d89 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Wed, 12 Jul 2017 14:55:42 +0100
Subject: [PATCH 022/115] Fix gcc error on JSON compilation

---
 lib/serialisation/JSON_IO.cc | 11 +++++++----
 lib/serialisation/JSON_IO.h  |  3 ++-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/serialisation/JSON_IO.cc b/lib/serialisation/JSON_IO.cc
index 303dfcb7..23a78b3e 100644
--- a/lib/serialisation/JSON_IO.cc
+++ b/lib/serialisation/JSON_IO.cc
@@ -69,7 +69,7 @@ void JSONWriter::delete_comma()
   ss_.str(dlast);
 }
 
-/*
+
 // here we are hitting a g++ bug (Bug 56480)
 // compiles fine with clang
 // have to wrap in the Grid namespace
@@ -79,13 +79,16 @@ namespace Grid
   template<>
   void JSONWriter::writeDefault(const std::string &s,	const std::string &x)
   {
+  //std::cout << "JSONWriter::writeDefault(string) : " << s <<  std::endl;
+    std::ostringstream os;
+    os << std::boolalpha << x;
     if (s.size())
-      ss_ << "\""<< s << "\" : \"" << x << "\" ," ; 
+      ss_ << "\""<< s << "\" : \"" << os.str() << "\" ," ;
     else
-      ss_ << "\"" << x << "\" ," ;
+     ss_ << os.str() << " ," ;
   }
 }// namespace Grid 
-*/
+
 
 // Reader implementation ///////////////////////////////////////////////////////
 JSONReader::JSONReader(const string &fileName)
diff --git a/lib/serialisation/JSON_IO.h b/lib/serialisation/JSON_IO.h
index a6d54fb3..c82648fc 100644
--- a/lib/serialisation/JSON_IO.h
+++ b/lib/serialisation/JSON_IO.h
@@ -120,6 +120,7 @@ namespace Grid
      ss_ << os.str() << " ," ;
   }
 
+/*
   // specialize for string
   template <>
   void JSONWriter::writeDefault(const std::string &s, const std::string &x)
@@ -132,7 +133,7 @@ namespace Grid
     else
      ss_ << os.str() << " ," ;
   }
-
+*/
 
 
   template <typename U>

From 7a53dc3715fbf2606aa1eb8ce943be35367e8ebb Mon Sep 17 00:00:00 2001
From: Nils Meyer <nils.meyer@ur.de>
Date: Mon, 24 Jul 2017 11:12:59 +0200
Subject: [PATCH 023/115] Added integer reduce functionality

---
 lib/simd/Grid_neon.h | 53 +++++++++++++++++---------------------------
 1 file changed, 20 insertions(+), 33 deletions(-)

diff --git a/lib/simd/Grid_neon.h b/lib/simd/Grid_neon.h
index 38815389..cadb4df8 100644
--- a/lib/simd/Grid_neon.h
+++ b/lib/simd/Grid_neon.h
@@ -6,9 +6,9 @@
 
     Copyright (C) 2015
 
-Author: Nils Meyer <nils.meyer@ur.de>
-Author: Peter Boyle <paboyle@ph.ed.ac.uk>
-Author: neo <cossu@post.kek.jp>
+    Author: Nils Meyer <nils.meyer@ur.de>
+    Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+    Author: neo <cossu@post.kek.jp>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -27,7 +27,7 @@ Author: neo <cossu@post.kek.jp>
     See the full license in the file "LICENSE" in the top level distribution directory
     *************************************************************************************/
     /*  END LEGAL */
-//----------------------------------------------------------------------
+
 /*
 
   ARMv8 NEON intrinsics layer by
@@ -37,9 +37,6 @@ Author: neo <cossu@post.kek.jp>
   SFB/TRR55
 
 */
-//----------------------------------------------------------------------
-//#ifndef ARM_NEON
-//#define ARM_NEON
 
 #ifndef GEN_SIMD_WIDTH
 #define GEN_SIMD_WIDTH 16u
@@ -85,11 +82,11 @@ namespace Optimization {
       double tmp[2]={a,b};
       return vld1q_f64(tmp);
     }
-    //Real double // N:tbc
+    //Real double
     inline float64x2_t operator()(double a){
       return vdupq_n_f64(a);
     }
-    //Integer // N:tbc
+    //Integer
     inline uint32x4_t operator()(Integer a){
       return vdupq_n_u32(a);
     }
@@ -127,33 +124,32 @@ namespace Optimization {
   // Nils: Vset untested; not used currently in Grid at all;
   // git commit 4a8c4ccfba1d05159348d21a9698028ea847e77b
   struct Vset{
-    // Complex float // N:ok
+    // Complex float
     inline float32x4_t operator()(Grid::ComplexF *a){
       float tmp[4]={a[1].imag(),a[1].real(),a[0].imag(),a[0].real()};
       return vld1q_f32(tmp);
     }
-    // Complex double // N:ok
+    // Complex double
     inline float64x2_t operator()(Grid::ComplexD *a){
       double tmp[2]={a[0].imag(),a[0].real()};
       return vld1q_f64(tmp);
     }
-    // Real float // N:ok
+    // Real float
     inline float32x4_t operator()(float *a){
       float tmp[4]={a[3],a[2],a[1],a[0]};
       return vld1q_f32(tmp);
     }
-    // Real double // N:ok
+    // Real double
     inline float64x2_t operator()(double *a){
       double tmp[2]={a[1],a[0]};
       return vld1q_f64(tmp);
     }
-    // Integer // N:ok
+    // Integer
     inline uint32x4_t operator()(Integer *a){
       return vld1q_dup_u32(a);
     }
   };
 
-  // N:leaving as is
   template <typename Out_type, typename In_type>
   struct Reduce{
     //Need templated class to overload output type
@@ -252,9 +248,9 @@ namespace Optimization {
       return vfmaq_f32(r4, r0, a); //  ar*br-ai*bi ai*br+ar*bi ...
 
       // no fma, use mul and add
-      //float32x4_t r5;
-      //r5 = vmulq_f32(r0, a);
-      //return vaddq_f32(r4, r5);
+      // float32x4_t r5;
+      // r5 = vmulq_f32(r0, a);
+      // return vaddq_f32(r4, r5);
     }
     // Complex double
     inline float64x2_t operator()(float64x2_t a, float64x2_t b){
@@ -275,9 +271,9 @@ namespace Optimization {
       return vfmaq_f64(r4, r0, a); //  ar*br-ai*bi ai*br+ar*bi
 
       // no fma, use mul and add
-      //float64x2_t r5;
-      //r5 = vmulq_f64(r0, a);
-      //return vaddq_f64(r4, r5);
+      // float64x2_t r5;
+      // r5 = vmulq_f64(r0, a);
+      // return vaddq_f64(r4, r5);
     }
   };
 
@@ -424,11 +420,6 @@ namespace Optimization {
       }
     }
 
-// working, but no restriction on n
-//    template<int n> static inline float32x4_t tRotate(float32x4_t in){ return vextq_f32(in,in,n); };
-//    template<int n> static inline float64x2_t tRotate(float64x2_t in){ return vextq_f64(in,in,n); };
-
-// restriction on n
     template<int n> static inline float32x4_t tRotate(float32x4_t in){ return vextq_f32(in,in,n%4); };
     template<int n> static inline float64x2_t tRotate(float64x2_t in){ return vextq_f64(in,in,n%2); };
 
@@ -444,7 +435,7 @@ namespace Optimization {
       sb = vcvt_high_f32_f16(h);
       // there is no direct conversion from lower float32x4_t to float64x2_t
       // vextq_f16 not supported by clang 3.8 / 4.0 / arm clang
-      //float16x8_t h1 = vextq_f16(h, h, 4); // correct, but not supported by clang
+      // float16x8_t h1 = vextq_f16(h, h, 4); // correct, but not supported by clang
       // workaround for clang
       uint32x4_t h1u = reinterpret_cast<uint32x4_t>(h);
       float16x8_t h1 = reinterpret_cast<float16x8_t>(vextq_u32(h1u, h1u, 2));
@@ -550,7 +541,7 @@ namespace Optimization {
 
 
   //Complex double Reduce
-  template<> // N:by Boyle
+  template<>
   inline Grid::ComplexD Reduce<Grid::ComplexD, float64x2_t>::operator()(float64x2_t in){
     u128d conv; conv.v = in;
     return Grid::ComplexD(conv.f[0],conv.f[1]);
@@ -565,9 +556,7 @@ namespace Optimization {
   //Integer Reduce
   template<>
   inline Integer Reduce<Integer, uint32x4_t>::operator()(uint32x4_t in){
-    // FIXME unimplemented
-    printf("Reduce : Missing integer implementation -> FIX\n");
-    assert(0);
+    return vaddvq_u32(in);
   }
 }
 
@@ -607,5 +596,3 @@ namespace Optimization {
   typedef Optimization::TimesI      TimesISIMD;
 
 }
-
-//#endif // ARM_NEON

From 9f280b82c4e56d8b32034dfbb83187e15add41e9 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Tue, 25 Jul 2017 11:30:41 -0400
Subject: [PATCH 024/115] Added mixed-precision CG with reliable updates

---
 lib/algorithms/Algorithms.h                   |   1 +
 .../ConjugateGradientReliableUpdate.h         | 231 ++++++++++++++++++
 tests/Test_dwf_mixedcg_prec_halfcomms.cc      |  40 ++-
 3 files changed, 260 insertions(+), 12 deletions(-)
 create mode 100644 lib/algorithms/iterative/ConjugateGradientReliableUpdate.h

diff --git a/lib/algorithms/Algorithms.h b/lib/algorithms/Algorithms.h
index 5123c7a1..f8dc2dc2 100644
--- a/lib/algorithms/Algorithms.h
+++ b/lib/algorithms/Algorithms.h
@@ -44,6 +44,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/algorithms/iterative/SchurRedBlack.h>
 #include <Grid/algorithms/iterative/ConjugateGradientMultiShift.h>
 #include <Grid/algorithms/iterative/ConjugateGradientMixedPrec.h>
+#include <Grid/algorithms/iterative/ConjugateGradientReliableUpdate.h>
 
 // Lanczos support
 //#include <Grid/algorithms/iterative/MatrixUtils.h>
diff --git a/lib/algorithms/iterative/ConjugateGradientReliableUpdate.h b/lib/algorithms/iterative/ConjugateGradientReliableUpdate.h
new file mode 100644
index 00000000..1aab064d
--- /dev/null
+++ b/lib/algorithms/iterative/ConjugateGradientReliableUpdate.h
@@ -0,0 +1,231 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/algorithms/iterative/ConjugateGradientReliableUpdate.h
+
+    Copyright (C) 2015
+
+Author: Christopher Kelly <ckelly@phys.columbia.edu>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef GRID_CONJUGATE_GRADIENT_RELIABLE_UPDATE_H
+#define GRID_CONJUGATE_GRADIENT_RELIABLE_UPDATE_H
+
+namespace Grid {
+
+  template<class FieldD,class FieldF, typename std::enable_if< getPrecision<FieldD>::value == 2, int>::type = 0,typename std::enable_if< getPrecision<FieldF>::value == 1, int>::type = 0> 
+  class ConjugateGradientReliableUpdate : public LinearFunction<FieldD> {
+  public:
+    bool ErrorOnNoConverge;  // throw an assert when the CG fails to converge.
+    // Defaults true.
+    RealD Tolerance;
+    Integer MaxIterations;
+    Integer IterationsToComplete; //Number of iterations the CG took to finish. Filled in upon completion
+    Integer ReliableUpdatesPerformed;
+
+    bool DoFinalCleanup; //Final DP cleanup, defaults to true
+    Integer IterationsToCleanup; //Final DP cleanup step iterations
+    
+    LinearOperatorBase<FieldF> &Linop_f;
+    LinearOperatorBase<FieldD> &Linop_d;
+    GridBase* SinglePrecGrid;
+    RealD Delta; //reliable update parameter
+    
+    ConjugateGradientReliableUpdate(RealD tol, Integer maxit, RealD _delta, GridBase* _sp_grid, LinearOperatorBase<FieldF> &_Linop_f, LinearOperatorBase<FieldD> &_Linop_d, bool err_on_no_conv = true)
+      : Tolerance(tol),
+        MaxIterations(maxit),
+	Delta(_delta),
+	Linop_f(_Linop_f),
+	Linop_d(_Linop_d),
+	SinglePrecGrid(_sp_grid),
+        ErrorOnNoConverge(err_on_no_conv),
+	DoFinalCleanup(true)
+    {};
+
+    void operator()(const FieldD &src, FieldD &psi) {
+      psi.checkerboard = src.checkerboard;
+      conformable(psi, src);
+
+      RealD cp, c, a, d, b, ssq, qq, b_pred;
+
+      FieldD p(src);
+      FieldD mmp(src);
+      FieldD r(src);
+
+      // Initial residual computation & set up
+      RealD guess = norm2(psi);
+      assert(std::isnan(guess) == 0);
+    
+      Linop_d.HermOpAndNorm(psi, mmp, d, b);
+    
+      r = src - mmp;
+      p = r;
+
+      a = norm2(p);
+      cp = a;
+      ssq = norm2(src);
+
+      std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate: guess " << guess << std::endl;
+      std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate:   src " << ssq << std::endl;
+      std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate:    mp " << d << std::endl;
+      std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate:   mmp " << b << std::endl;
+      std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate:  cp,r " << cp << std::endl;
+      std::cout << GridLogIterative << std::setprecision(4) << "ConjugateGradientReliableUpdate:     p " << a << std::endl;
+
+      RealD rsq = Tolerance * Tolerance * ssq;
+
+      // Check if guess is really REALLY good :)
+      if (cp <= rsq) {
+	return;
+      }
+
+      //Single prec initialization
+      FieldF r_f(SinglePrecGrid);
+      r_f.checkerboard = r.checkerboard;
+      precisionChange(r_f, r);
+
+      FieldF psi_f(r_f);
+      psi_f = zero;
+
+      FieldF p_f(r_f);
+      FieldF mmp_f(r_f);
+
+      RealD MaxResidSinceLastRelUp = cp; //initial residual    
+    
+      std::cout << GridLogIterative << std::setprecision(4)
+		<< "ConjugateGradient: k=0 residual " << cp << " target " << rsq << std::endl;
+
+      GridStopWatch LinalgTimer;
+      GridStopWatch MatrixTimer;
+      GridStopWatch SolverTimer;
+
+      SolverTimer.Start();
+      int k = 0;
+      int l = 0;
+    
+      for (k = 1; k <= MaxIterations; k++) {
+	c = cp;
+
+	MatrixTimer.Start();
+	Linop_f.HermOpAndNorm(p_f, mmp_f, d, qq);
+	MatrixTimer.Stop();
+
+	LinalgTimer.Start();
+
+	a = c / d;
+	b_pred = a * (a * qq - d) / c;
+
+	cp = axpy_norm(r_f, -a, mmp_f, r_f);
+	b = cp / c;
+
+	// Fuse these loops ; should be really easy
+	psi_f = a * p_f + psi_f;
+	//p_f = p_f * b + r_f;
+
+	LinalgTimer.Stop();
+
+	std::cout << GridLogIterative << "ConjugateGradientReliableUpdate: Iteration " << k
+		  << " residual " << cp << " target " << rsq << std::endl;
+	std::cout << GridLogDebug << "a = "<< a << " b_pred = "<< b_pred << "  b = "<< b << std::endl;
+	std::cout << GridLogDebug << "qq = "<< qq << " d = "<< d << "  c = "<< c << std::endl;
+
+	if(cp > MaxResidSinceLastRelUp){
+	  std::cout << GridLogIterative << "ConjugateGradientReliableUpdate: updating MaxResidSinceLastRelUp : " << MaxResidSinceLastRelUp << " -> " << cp << std::endl;
+	  MaxResidSinceLastRelUp = cp;
+	}
+	  
+	// Stopping condition
+	if (cp <= rsq) {
+	  //Although not written in the paper, I assume that I have to add on the final solution
+	  precisionChange(mmp, psi_f);
+	  psi = psi + mmp;
+	
+	
+	  SolverTimer.Stop();
+	  Linop_d.HermOpAndNorm(psi, mmp, d, qq);
+	  p = mmp - src;
+
+	  RealD srcnorm = sqrt(norm2(src));
+	  RealD resnorm = sqrt(norm2(p));
+	  RealD true_residual = resnorm / srcnorm;
+
+	  std::cout << GridLogMessage << "ConjugateGradientReliableUpdate Converged on iteration " << k << " after " << l << " reliable updates" << std::endl;
+	  std::cout << GridLogMessage << "\tComputed residual " << sqrt(cp / ssq)<<std::endl;
+	  std::cout << GridLogMessage << "\tTrue residual " << true_residual<<std::endl;
+	  std::cout << GridLogMessage << "\tTarget " << Tolerance << std::endl;
+
+	  std::cout << GridLogMessage << "Time breakdown "<<std::endl;
+	  std::cout << GridLogMessage << "\tElapsed    " << SolverTimer.Elapsed() <<std::endl;
+	  std::cout << GridLogMessage << "\tMatrix     " << MatrixTimer.Elapsed() <<std::endl;
+	  std::cout << GridLogMessage << "\tLinalg     " << LinalgTimer.Elapsed() <<std::endl;
+
+	  IterationsToComplete = k;	
+	  ReliableUpdatesPerformed = l;
+	  
+	  if(DoFinalCleanup){
+	    //Do a final CG to cleanup
+	    std::cout << GridLogMessage << "ConjugateGradientReliableUpdate performing final cleanup.\n";
+	    ConjugateGradient<FieldD> CG(Tolerance,MaxIterations);
+	    CG.ErrorOnNoConverge = ErrorOnNoConverge;
+	    CG(Linop_d,src,psi);
+	    IterationsToCleanup = CG.IterationsToComplete;
+	  }
+	  else if (ErrorOnNoConverge) assert(true_residual / Tolerance < 10000.0);
+
+	  std::cout << GridLogMessage << "ConjugateGradientReliableUpdate complete.\n";
+	  return;
+	}
+	else if(cp < Delta * MaxResidSinceLastRelUp) { //reliable update
+	  std::cout << GridLogMessage << "ConjugateGradientReliableUpdate "
+		    << cp << "(residual) < " << Delta << "(Delta) * " << MaxResidSinceLastRelUp << "(MaxResidSinceLastRelUp) on iteration " << k << " : performing reliable update\n";
+	  precisionChange(mmp, psi_f);
+	  psi = psi + mmp;
+
+	  Linop_d.HermOpAndNorm(psi, mmp, d, qq);
+	  r = src - mmp;
+
+	  psi_f = zero;
+	  precisionChange(r_f, r);
+	  cp = norm2(r);
+	  MaxResidSinceLastRelUp = cp;
+
+	  std::cout << GridLogMessage << "ConjugateGradientReliableUpdate new residual " << cp << std::endl;
+	  
+	  l = l+1;
+	}
+
+	p_f = p_f * b + r_f; //update search vector after reliable update appears to help convergence
+	
+      }
+      std::cout << GridLogMessage << "ConjugateGradientReliableUpdate did NOT converge"
+		<< std::endl;
+      
+      if (ErrorOnNoConverge) assert(0);
+      IterationsToComplete = k;
+      ReliableUpdatesPerformed = l;      
+    }    
+  };
+
+
+};
+
+
+
+#endif
diff --git a/tests/Test_dwf_mixedcg_prec_halfcomms.cc b/tests/Test_dwf_mixedcg_prec_halfcomms.cc
index 9cc935d9..d6aaa21e 100644
--- a/tests/Test_dwf_mixedcg_prec_halfcomms.cc
+++ b/tests/Test_dwf_mixedcg_prec_halfcomms.cc
@@ -80,31 +80,47 @@ int main (int argc, char ** argv)
 
 
   LatticeFermionD    src_o(FrbGrid);
-  LatticeFermionD result_o(FrbGrid);
-  LatticeFermionD result_o_2(FrbGrid);
+  LatticeFermionD result_cg(FrbGrid);
   pickCheckerboard(Odd,src_o,src);
-  result_o.checkerboard = Odd;
-  result_o = zero;
-  result_o_2.checkerboard = Odd;
-  result_o_2 = zero;
+  result_cg.checkerboard = Odd;
+  result_cg = zero;
+  LatticeFermionD result_mcg(result_cg);
+  LatticeFermionD result_rlcg(result_cg);
 
   SchurDiagMooeeOperator<DomainWallFermionD,LatticeFermionD> HermOpEO(Ddwf);
   SchurDiagMooeeOperator<DomainWallFermionFH,LatticeFermionF> HermOpEO_f(Ddwf_f);
 
+  //#define DO_MIXED_CG
+#define DO_RLUP_CG
+
+#ifdef DO_MIXED_CG
   std::cout << "Starting mixed CG" << std::endl;
   MixedPrecisionConjugateGradient<LatticeFermionD,LatticeFermionF> mCG(1.0e-8, 10000, 50, FrbGrid_f, HermOpEO_f, HermOpEO);
   mCG.InnerTolerance = 3.0e-5;
-  mCG(src_o,result_o);
+  mCG(src_o,result_mcg);
+#endif
 
+#ifdef DO_RLUP_CG
+  std::cout << "Starting reliable update CG" << std::endl;
+  ConjugateGradientReliableUpdate<LatticeFermionD,LatticeFermionF> rlCG(1.e-8, 10000, 0.1, FrbGrid_f, HermOpEO_f, HermOpEO);
+  rlCG(src_o,result_rlcg);
+#endif
+  
   std::cout << "Starting regular CG" << std::endl;
   ConjugateGradient<LatticeFermionD> CG(1.0e-8,10000);
-  CG(HermOpEO,src_o,result_o_2);
+  CG(HermOpEO,src_o,result_cg);
 
-  LatticeFermionD diff_o(FrbGrid);
-  RealD diff = axpy_norm(diff_o, -1.0, result_o, result_o_2);
-
-  std::cout << "Diff between mixed and regular CG: " << diff << std::endl;
+#ifdef DO_MIXED_CG
+  LatticeFermionD diff_mcg(FrbGrid);
+  RealD vdiff_mcg = axpy_norm(diff_mcg, -1.0, result_cg, result_mcg);
+  std::cout << "Diff between mixed and regular CG: " << vdiff_mcg << std::endl;
+#endif
 
+#ifdef DO_RLUP_CG
+  LatticeFermionD diff_rlcg(FrbGrid);
+  RealD vdiff_rlcg = axpy_norm(diff_rlcg, -1.0, result_cg, result_rlcg);
+  std::cout << "Diff between reliable update and regular CG: " << vdiff_rlcg << std::endl;
+#endif
   
   Grid_finalize();
 }

From 14d53e1c9eb8eb1ef684148728c075813814612e Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sat, 29 Jul 2017 13:06:53 -0400
Subject: [PATCH 025/115] Threaded MPI calls patches

---
 benchmarks/Benchmark_dwf.cc               |  2 +-
 lib/allocator/AlignedAllocator.h          | 10 ++-
 lib/communicator/Communicator_base.cc     |  4 +-
 lib/communicator/Communicator_base.h      | 14 ++++-
 lib/communicator/Communicator_mpit.cc     | 25 +++++++-
 lib/qcd/action/fermion/WilsonFermion5D.cc | 74 ++++++++++++++---------
 lib/stencil/Stencil.h                     | 59 +++++++++---------
 lib/util/Init.cc                          |  6 +-
 8 files changed, 128 insertions(+), 66 deletions(-)

diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index a071c050..0264905c 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -489,7 +489,7 @@ int main (int argc, char ** argv)
 
   //assert(norm2(src_e)<1.0e-4);
   //assert(norm2(src_o)<1.0e-4);
-
+  exit(0);
   Grid_finalize();
 }
 
diff --git a/lib/allocator/AlignedAllocator.h b/lib/allocator/AlignedAllocator.h
index 6e85ab27..7fd9496f 100644
--- a/lib/allocator/AlignedAllocator.h
+++ b/lib/allocator/AlignedAllocator.h
@@ -92,11 +92,15 @@ public:
     size_type bytes = __n*sizeof(_Tp);
 
     _Tp *ptr = (_Tp *) PointerCache::Lookup(bytes);
-    
+    //////////////////
+    // Hack 2MB align; could make option probably doesn't need configurability
+    //////////////////
+//define GRID_ALLOC_ALIGN (128)
+#define GRID_ALLOC_ALIGN (2*1024*1024)
 #ifdef HAVE_MM_MALLOC_H
-    if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) _mm_malloc(bytes,128);
+    if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) _mm_malloc(bytes,GRID_ALLOC_ALIGN);
 #else
-    if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) memalign(128,bytes);
+    if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) memalign(GRID_ALLOC_ALIGN,bytes);
 #endif
 
     return ptr;
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index a5edf8e9..67bfaed0 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -34,7 +34,9 @@ namespace Grid {
 ///////////////////////////////////////////////////////////////
 void *              CartesianCommunicator::ShmCommBuf;
 uint64_t            CartesianCommunicator::MAX_MPI_SHM_BYTES   = 128*1024*1024; 
-CartesianCommunicator::CommunicatorPolicy_t  CartesianCommunicator::CommunicatorPolicy= CartesianCommunicator::CommunicatorPolicyConcurrent;
+CartesianCommunicator::CommunicatorPolicy_t  
+CartesianCommunicator::CommunicatorPolicy= CartesianCommunicator::CommunicatorPolicyConcurrent;
+int CartesianCommunicator::nCommThreads = -1;
 
 /////////////////////////////////
 // Alloc, free shmem region
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 4e471b43..84dbedb4 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -54,8 +54,9 @@ class CartesianCommunicator {
   // 128MB shared memory for comms enought for 48^4 local vol comms
   // Give external control (command line override?) of this
 
-  static const int      MAXLOG2RANKSPERNODE = 16;            
-  static uint64_t MAX_MPI_SHM_BYTES;
+  static const int MAXLOG2RANKSPERNODE = 16;            
+  static uint64_t  MAX_MPI_SHM_BYTES;
+  static int       nCommThreads;
 
   // Communicator should know nothing of the physics grid, only processor grid.
   int              _Nprocessors;     // How many in all
@@ -125,7 +126,7 @@ class CartesianCommunicator {
   enum CommunicatorPolicy_t { CommunicatorPolicyConcurrent, CommunicatorPolicySequential };
   static CommunicatorPolicy_t CommunicatorPolicy;
   static void SetCommunicatorPolicy(CommunicatorPolicy_t policy ) { CommunicatorPolicy = policy; }
-
+  
   size_t heap_top;
   size_t heap_bytes;
 
@@ -215,6 +216,12 @@ class CartesianCommunicator {
   
   void SendToRecvFromComplete(std::vector<CommsRequest_t> &waitall);
 
+  double StencilSendToRecvFrom(void *xmit,
+			       int xmit_to_rank,
+			       void *recv,
+			       int recv_from_rank,
+			       int bytes,int dir);
+
   double StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 				    void *xmit,
 				    int xmit_to_rank,
@@ -222,6 +229,7 @@ class CartesianCommunicator {
 				    int recv_from_rank,
 				    int bytes,int dir);
   
+  
   void StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int i);
   void StencilBarrier(void);
 
diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc
index 24a518ec..f522701c 100644
--- a/lib/communicator/Communicator_mpit.cc
+++ b/lib/communicator/Communicator_mpit.cc
@@ -242,7 +242,8 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 							 int recv_from_rank,
 							 int bytes,int dir)
 {
-
+  assert(false);
+  /*
   int myrank = _processor;
   int ierr;
   assert(dir < communicator_halo.size());
@@ -254,6 +255,28 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 		    communicator_halo[dir],MPI_STATUS_IGNORE);
   assert(ierr==0);
   return 2.0*bytes;
+  */
+}
+
+double CartesianCommunicator::StencilSendToRecvFrom(void *xmit,
+						    int xmit_to_rank,
+						    void *recv,
+						    int recv_from_rank,
+						    int bytes,int dir)
+{
+  int myrank = _processor;
+  int ierr;
+  assert(dir < communicator_halo.size());
+  
+  //  std::cout << " sending on communicator "<<dir<<" " <<communicator_halo[dir]<<std::endl;
+  // Give the CPU to MPI immediately; can use threads to overlap optionally
+  MPI_Request req[2];
+  MPI_Irecv(recv,bytes,MPI_CHAR,recv_from_rank,recv_from_rank,
+	    communicator_halo[dir],&req[1]);
+  MPI_Isend(xmit,bytes,MPI_CHAR,xmit_to_rank,myrank,
+	    communicator_halo[dir], &req[0]);
+  MPI_Waitall(2, req, MPI_STATUSES_IGNORE);
+  return 2.0*bytes;
 }
 void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
 { 
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index 6a6bc1f8..0b6c9e3d 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -391,37 +391,57 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
   DhopFaceTime+=usecond();
 
   // Rely on async comms; start comms before merge of local data
-  DhopComputeTime-=usecond();
-  DhopCommTime-=usecond();
-#pragma omp parallel 
+  double ctime=0;
+  double ptime=0;
+  //  DhopComputeTime-=usecond();
+  //  DhopCommTime-=usecond();
+#pragma omp parallel reduction(max:ctime) reduction(max:ptime)
   { 
-    // Should time this somehow; hard as the threads fork nowait
-    st.CommunicateThreaded();
-
-  if (dag == DaggerYes) {
-#pragma omp for
-    for (int ss = 0; ss < U._grid->oSites(); ss++) {
-      int sU = ss;
-      int sF = LLs * sU;
-      Kernels::DhopSiteDag(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,1,0);
+    int tid = omp_get_thread_num();
+    int nthreads = omp_get_num_threads();
+    int ncomms = CartesianCommunicator::nCommThreads;
+    if (ncomms == -1) ncomms = st.Packets.size(); 
+    assert(nthreads > ncomms);
+    if (tid >= ncomms) {
+      double start = usecond();
+      nthreads -= ncomms;
+      int ttid = tid - ncomms;
+      int n = U._grid->oSites();
+      int chunk = n / nthreads;
+      int rem = n % nthreads;
+      int myblock, myn;
+      if (ttid < rem) {
+	myblock = ttid * chunk + ttid;
+	myn = chunk+1;
+      } else {
+	myblock = ttid*chunk + rem;
+	myn = chunk;
+      }
+      
+      // do the compute
+      if (dag == DaggerYes) {
+	for (int ss = myblock; ss < myblock+myn; ++ss) {
+	  int sU = ss;
+	  int sF = LLs * sU;
+	  Kernels::DhopSiteDag(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,1,0);
+	}
+      } else {
+	for (int ss = myblock; ss < myblock+myn; ++ss) {
+	  int sU = ss;
+	  int sF = LLs * sU;
+	  Kernels::DhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,1,0);
+	}
+      }
+	ptime = usecond() - start;
     }
-  } else {
-#pragma omp for
-    for (int ss = 0; ss < U._grid->oSites(); ss++) {
-      int sU = ss;
-      int sF = LLs * sU;
-      Kernels::DhopSite(st,lo,U,st.CommBuf(),sF,sU,LLs,1,in,out,1,0);
+    {
+      double start = usecond();
+      st.CommunicateThreaded();
+      ctime = usecond() - start;
     }
   }
-#pragma omp single
-  DhopComputeTime+=usecond();
-
-#pragma omp taskwait 
-
-#pragma omp single
-  DhopCommTime+=usecond();
-  } // Closes parallel region and waits the comms (I hope)
-
+  DhopCommTime += ctime;
+  DhopComputeTime+=ptime;
 
   DhopFaceTime-=usecond();
   st.CommsMerge(compressor);
diff --git a/lib/stencil/Stencil.h b/lib/stencil/Stencil.h
index 17db64d8..d1d7a7e0 100644
--- a/lib/stencil/Stencil.h
+++ b/lib/stencil/Stencil.h
@@ -185,6 +185,8 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
   double splicetime;
   double nosplicetime;
   double calls;
+  std::vector<double> comms_bytesthr;
+  std::vector<double> commtimethr;
 
   ////////////////////////////////////////
   // Stencil query
@@ -250,36 +252,22 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
   //////////////////////////////////////////
   void CommunicateThreaded()
   {
-    for(int i=0;i<Packets.size();i++){
-#pragma omp task 
-      {
-	double start;
-	double stop;
-	start = usecond();
-	uint64_t bytes;
-	std::vector<CommsRequest_t> reqs;
-	bytes=_grid->StencilSendToRecvFromBegin(reqs,
-					  Packets[i].send_buf,
-					  Packets[i].to_rank,
-					  Packets[i].recv_buf,
-					  Packets[i].from_rank,
-					  Packets[i].bytes,i);
-	_grid->StencilSendToRecvFromComplete(reqs,i);
-	// Last task logged; this is approximate but hard to catch
-	// the last to complete
-	stop = usecond();
-	stop = stop - start;
-
-	if ( i==0 ) commtime+=stop;
-
-#pragma omp critical
-	{
-	  comms_bytes+=bytes;
-	}
-
+    // must be called in parallel region
+    int mythread = omp_get_thread_num();
+    int nthreads = CartesianCommunicator::nCommThreads;
+    if (nthreads == -1) nthreads = Packets.size();
+    if (mythread < nthreads) {
+      for (int i = mythread; i < Packets.size(); i += nthreads) {
+	double start = usecond();
+	uint64_t bytes = _grid->StencilSendToRecvFrom(Packets[i].send_buf,
+						      Packets[i].to_rank,
+						      Packets[i].recv_buf,
+						      Packets[i].from_rank,
+						      Packets[i].bytes,i);
+	comms_bytesthr[mythread] += bytes;
+	commtimethr[mythread] += usecond() - start;
       }
     }
-    
   }
   void CommunicateBegin(std::vector<std::vector<CommsRequest_t> > &reqs)
   {
@@ -475,7 +463,10 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
 		  int checkerboard,
 		  const std::vector<int> &directions,
 		  const std::vector<int> &distances) 
-   :   _permute_type(npoints), _comm_buf_size(npoints)
+   : _permute_type(npoints), 
+    _comm_buf_size(npoints),
+    comms_bytesthr(npoints), 
+       commtimethr(npoints)
   {
     face_table_computed=0;
     _npoints = npoints;
@@ -1029,6 +1020,8 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
   void ZeroCounters(void) {
     gathertime = 0.;
     commtime = 0.;
+    memset(&commtimethr[0], 0, sizeof(commtimethr));
+    memset(&comms_bytesthr[0], 0, sizeof(comms_bytesthr));
     halogtime = 0.;
     mergetime = 0.;
     decompresstime = 0.;
@@ -1044,6 +1037,14 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
 #define PRINTIT(A) AVERAGE(A); std::cout << GridLogMessage << " Stencil " << #A << " "<< A/calls<<std::endl;
     RealD NP = _grid->_Nprocessors;
     RealD NN = _grid->NodeCount();
+    double t = 0;
+    // if commtimethr is set they were all done in parallel so take the max
+    // but add up the bytes
+    for (int i = 0; i < 8; ++i) {
+      comms_bytes += comms_bytesthr[i];
+      if (t < commtimethr[i]) t = commtimethr[i];
+    }
+    commtime += t;
     
     _grid->GlobalSum(commtime);    commtime/=NP;
     if ( calls > 0. ) {
diff --git a/lib/util/Init.cc b/lib/util/Init.cc
index fc701ac1..ef875429 100644
--- a/lib/util/Init.cc
+++ b/lib/util/Init.cc
@@ -359,7 +359,11 @@ void Grid_init(int *argc,char ***argv)
   if( GridCmdOptionExists(*argv,*argv+*argc,"--lebesgue") ){
     LebesgueOrder::UseLebesgueOrder=1;
   }
-
+  CartesianCommunicator::nCommThreads = -1;
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--commthreads") ){
+    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--commthreads");
+    GridCmdOptionInt(arg,CartesianCommunicator::nCommThreads);
+  }
   if( GridCmdOptionExists(*argv,*argv+*argc,"--cacheblocking") ){
     arg= GridCmdOptionPayload(*argv,*argv+*argc,"--cacheblocking");
     GridCmdOptionIntVector(arg,LebesgueOrder::Block);

From 9939b267d26f13cf5c152705741ea7655badece9 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Mon, 31 Jul 2017 13:39:44 -0400
Subject: [PATCH 026/115] Added switching to fallback linear operator in
 reliable update CG, and added recalculation of b parameter on update.

---
 .../ConjugateGradientReliableUpdate.h         | 29 +++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/lib/algorithms/iterative/ConjugateGradientReliableUpdate.h b/lib/algorithms/iterative/ConjugateGradientReliableUpdate.h
index 1aab064d..13666f97 100644
--- a/lib/algorithms/iterative/ConjugateGradientReliableUpdate.h
+++ b/lib/algorithms/iterative/ConjugateGradientReliableUpdate.h
@@ -47,6 +47,11 @@ namespace Grid {
     LinearOperatorBase<FieldD> &Linop_d;
     GridBase* SinglePrecGrid;
     RealD Delta; //reliable update parameter
+
+    //Optional ability to switch to a different linear operator once the tolerance reaches a certain point. Useful for single/half -> single/single
+    LinearOperatorBase<FieldF> *Linop_fallback;
+    RealD fallback_transition_tol;
+
     
     ConjugateGradientReliableUpdate(RealD tol, Integer maxit, RealD _delta, GridBase* _sp_grid, LinearOperatorBase<FieldF> &_Linop_f, LinearOperatorBase<FieldD> &_Linop_d, bool err_on_no_conv = true)
       : Tolerance(tol),
@@ -56,10 +61,19 @@ namespace Grid {
 	Linop_d(_Linop_d),
 	SinglePrecGrid(_sp_grid),
         ErrorOnNoConverge(err_on_no_conv),
-	DoFinalCleanup(true)
+	DoFinalCleanup(true),
+	Linop_fallback(NULL)
     {};
 
+    void setFallbackLinop(LinearOperatorBase<FieldF> &_Linop_fallback, const RealD _fallback_transition_tol){
+      Linop_fallback = &_Linop_fallback;
+      fallback_transition_tol = _fallback_transition_tol;      
+    }
+    
     void operator()(const FieldD &src, FieldD &psi) {
+      LinearOperatorBase<FieldF> *Linop_f_use = &Linop_f;
+      bool using_fallback = false;
+      
       psi.checkerboard = src.checkerboard;
       conformable(psi, src);
 
@@ -93,6 +107,8 @@ namespace Grid {
 
       // Check if guess is really REALLY good :)
       if (cp <= rsq) {
+	std::cout << GridLogMessage << "ConjugateGradientReliableUpdate guess was REALLY good\n";
+	std::cout << GridLogMessage << "\tComputed residual " << sqrt(cp / ssq)<<std::endl;
 	return;
       }
 
@@ -124,7 +140,7 @@ namespace Grid {
 	c = cp;
 
 	MatrixTimer.Start();
-	Linop_f.HermOpAndNorm(p_f, mmp_f, d, qq);
+	Linop_f_use->HermOpAndNorm(p_f, mmp_f, d, qq);
 	MatrixTimer.Stop();
 
 	LinalgTimer.Start();
@@ -206,12 +222,21 @@ namespace Grid {
 	  cp = norm2(r);
 	  MaxResidSinceLastRelUp = cp;
 
+	  b = cp/c;
+	  
 	  std::cout << GridLogMessage << "ConjugateGradientReliableUpdate new residual " << cp << std::endl;
 	  
 	  l = l+1;
 	}
 
 	p_f = p_f * b + r_f; //update search vector after reliable update appears to help convergence
+
+	if(!using_fallback && Linop_fallback != NULL && cp < fallback_transition_tol){
+	  std::cout << GridLogMessage << "ConjugateGradientReliableUpdate switching to fallback linear operator on iteration " << k << " at residual " << cp << std::endl;
+	  Linop_f_use = Linop_fallback;
+	  using_fallback = true;
+	}
+
 	
       }
       std::cout << GridLogMessage << "ConjugateGradientReliableUpdate did NOT converge"

From 4fe182e5a7c4b1d1dddc022706a71f1c0432cda5 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Sun, 6 Aug 2017 10:46:19 +0100
Subject: [PATCH 027/115] Added high level HMC support for overriding default
 SIMD lane decomposition

---
 lib/cartesian/Cartesian_base.h      | 23 ++++----
 lib/qcd/hmc/HMCResourceManager.h    | 14 ++++-
 lib/qcd/hmc/HMC_GridModules.h       | 92 +++++++++++++++++++++--------
 lib/util/Init.cc                    |  2 +-
 tests/hmc/Test_hmc_EOMobiusRatio.cc | 13 ++--
 5 files changed, 98 insertions(+), 46 deletions(-)

diff --git a/lib/cartesian/Cartesian_base.h b/lib/cartesian/Cartesian_base.h
index 0db6ce0d..f4f9a269 100644
--- a/lib/cartesian/Cartesian_base.h
+++ b/lib/cartesian/Cartesian_base.h
@@ -185,17 +185,18 @@ public:
     ////////////////////////////////////////////////////////////////
 
     void show_decomposition(){
-      std::cout << GridLogMessage << "Full Dimensions    : " << _fdimensions << std::endl;
-      std::cout << GridLogMessage << "Global Dimensions  : " << _gdimensions << std::endl;
-      std::cout << GridLogMessage << "Local Dimensions   : " << _ldimensions << std::endl;
-      std::cout << GridLogMessage << "Reduced Dimensions : " << _rdimensions << std::endl;
-      std::cout << GridLogMessage << "Outer strides      : " << _ostride << std::endl;
-      std::cout << GridLogMessage << "Inner strides      : " << _istride << std::endl;
-      std::cout << GridLogMessage << "iSites             : " << _isites << std::endl;
-      std::cout << GridLogMessage << "oSites             : " << _osites << std::endl;
-      std::cout << GridLogMessage << "lSites             : " << lSites() << std::endl;        
-      std::cout << GridLogMessage << "gSites             : " << gSites() << std::endl;
-      std::cout << GridLogMessage << "Nd                 : " << _ndimension << std::endl;             
+      std::cout << GridLogMessage << "\tFull Dimensions    : " << _fdimensions << std::endl;
+      std::cout << GridLogMessage << "\tSIMD layout        : " << _simd_layout << std::endl;
+      std::cout << GridLogMessage << "\tGlobal Dimensions  : " << _gdimensions << std::endl;
+      std::cout << GridLogMessage << "\tLocal Dimensions   : " << _ldimensions << std::endl;
+      std::cout << GridLogMessage << "\tReduced Dimensions : " << _rdimensions << std::endl;
+      std::cout << GridLogMessage << "\tOuter strides      : " << _ostride << std::endl;
+      std::cout << GridLogMessage << "\tInner strides      : " << _istride << std::endl;
+      std::cout << GridLogMessage << "\tiSites             : " << _isites << std::endl;
+      std::cout << GridLogMessage << "\toSites             : " << _osites << std::endl;
+      std::cout << GridLogMessage << "\tlSites             : " << lSites() << std::endl;        
+      std::cout << GridLogMessage << "\tgSites             : " << gSites() << std::endl;
+      std::cout << GridLogMessage << "\tNd                 : " << _ndimension << std::endl;             
     } 
 
     ////////////////////////////////////////////////////////////////
diff --git a/lib/qcd/hmc/HMCResourceManager.h b/lib/qcd/hmc/HMCResourceManager.h
index cf0000ed..3e20a8c1 100644
--- a/lib/qcd/hmc/HMCResourceManager.h
+++ b/lib/qcd/hmc/HMCResourceManager.h
@@ -165,7 +165,7 @@ class HMCResourceManager {
   // Grids
   //////////////////////////////////////////////////////////////
 
-  void AddGrid(std::string s, GridModule& M) {
+  void AddGrid(const std::string s, GridModule& M) {
     // Check for name clashes
     auto search = Grids.find(s);
     if (search != Grids.end()) {
@@ -174,14 +174,24 @@ class HMCResourceManager {
       exit(1);
     }
     Grids[s] = std::move(M);
+    std::cout << GridLogMessage << "::::::::::::::::::::::::::::::::::::::::" <<std::endl;
+    std::cout << GridLogMessage << "HMCResourceManager:" << std::endl;
+    std::cout << GridLogMessage << "Created grid set with name '" << s << "' and decomposition for the full cartesian " << std::endl;
+    Grids[s].show_full_decomposition();
+    std::cout << GridLogMessage << "::::::::::::::::::::::::::::::::::::::::" <<std::endl;
   }
 
   // Add a named grid set, 4d shortcut
-  void AddFourDimGrid(std::string s) {
+  void AddFourDimGrid(const std::string s) {
     GridFourDimModule<vComplex> Mod;
     AddGrid(s, Mod);
   }
 
+  // Add a named grid set, 4d shortcut + tweak simd lanes
+  void AddFourDimGrid(const std::string s, const std::vector<int> simd_decomposition) {
+    GridFourDimModule<vComplex> Mod(simd_decomposition);
+    AddGrid(s, Mod);
+  }
 
 
   GridCartesian* GetCartesian(std::string s = "") {
diff --git a/lib/qcd/hmc/HMC_GridModules.h b/lib/qcd/hmc/HMC_GridModules.h
index 8331c02b..0f34e9a7 100644
--- a/lib/qcd/hmc/HMC_GridModules.h
+++ b/lib/qcd/hmc/HMC_GridModules.h
@@ -33,28 +33,29 @@ directory
 namespace Grid {
 
 // Resources
-// Modules for grids 
+// Modules for grids
 
 // Introduce another namespace HMCModules?
 
-class GridModuleParameters: Serializable{   
+class GridModuleParameters: Serializable{
 public:
   GRID_SERIALIZABLE_CLASS_MEMBERS(GridModuleParameters,
   std::string, lattice,
   std::string, mpi);
 
-  std::vector<int> getLattice(){return strToVec<int>(lattice);}
-  std::vector<int> getMpi()    {return strToVec<int>(mpi);}
+  std::vector<int> getLattice() const {return strToVec<int>(lattice);}
+  std::vector<int> getMpi()     const {return strToVec<int>(mpi);}
 
-  void check(){
-    if (getLattice().size() != getMpi().size()) {
-      std::cout << GridLogError 
+
+  void check() const {
+    if (getLattice().size() != getMpi().size() ) {
+      std::cout << GridLogError
                 << "Error in GridModuleParameters: lattice and mpi dimensions "
                    "do not match"
                 << std::endl;
       exit(1);
     }
-  }    
+  }
 
   template <class ReaderClass>
   GridModuleParameters(Reader<ReaderClass>& Reader, std::string n = "LatticeGrid"):name(n) {
@@ -75,51 +76,94 @@ private:
 // Lower level class
 class GridModule {
  public:
-  GridCartesian* get_full() { 
+  GridCartesian* get_full() {
     std::cout << GridLogDebug << "Getting cartesian in module"<< std::endl;
     return grid_.get(); }
-  GridRedBlackCartesian* get_rb() { 
+  GridRedBlackCartesian* get_rb() {
     std::cout << GridLogDebug << "Getting rb-cartesian in module"<< std::endl;
     return rbgrid_.get(); }
 
   void set_full(GridCartesian* grid) { grid_.reset(grid); }
   void set_rb(GridRedBlackCartesian* rbgrid) { rbgrid_.reset(rbgrid); }
+  void show_full_decomposition(){ grid_->show_decomposition(); }
+  void show_rb_decomposition(){ rbgrid_->show_decomposition(); }
 
  protected:
   std::unique_ptr<GridCartesian> grid_;
   std::unique_ptr<GridRedBlackCartesian> rbgrid_;
-  
+
 };
 
 ////////////////////////////////////
 // Classes for the user
 ////////////////////////////////////
 // Note: the space time grid should be out of the QCD namespace
-template< class vector_type>
-class GridFourDimModule : public GridModule {
- public:
-  GridFourDimModule() {
+template <class vector_type>
+class GridFourDimModule : public GridModule
+{
+public:
+  GridFourDimModule()
+  {
     using namespace QCD;
     set_full(SpaceTimeGrid::makeFourDimGrid(
-        GridDefaultLatt(), GridDefaultSimd(4, vector_type::Nsimd()),
+        GridDefaultLatt(), 
+        GridDefaultSimd(4, vector_type::Nsimd()),
         GridDefaultMpi()));
     set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get()));
   }
 
-  GridFourDimModule(GridModuleParameters Params) {
+  GridFourDimModule(const std::vector<int> tweak_simd)
+  {
+    using namespace QCD;
+    if (tweak_simd.size() != 4)
+    {
+      std::cout << GridLogError
+                << "Error in GridFourDimModule: SIMD size different from 4" 
+                << std::endl;
+      exit(1);
+    }
+
+    // Checks that the product agrees with the expectation
+    int simd_sum = 1;
+    for (auto &n : tweak_simd)
+      simd_sum *= n;
+    std::cout << GridLogDebug << "TweakSIMD: " << tweak_simd << "  Sum: " << simd_sum << std::endl;
+
+    if (simd_sum == vector_type::Nsimd())
+    {
+      set_full(SpaceTimeGrid::makeFourDimGrid(
+          GridDefaultLatt(), 
+          tweak_simd, 
+          GridDefaultMpi()));
+      set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get()));
+    }
+    else
+    {
+      std::cout << GridLogError 
+                << "Error in GridFourDimModule: SIMD lanes must sum to " 
+                << vector_type::Nsimd() 
+                << std::endl;
+    }
+  }
+
+  GridFourDimModule(const GridModuleParameters Params)
+  {
     using namespace QCD;
-    Params.check();
     std::vector<int> lattice_v = Params.getLattice();
     std::vector<int> mpi_v = Params.getMpi();
-    if (lattice_v.size() == 4) {
+    if (lattice_v.size() == 4)
+    {
       set_full(SpaceTimeGrid::makeFourDimGrid(
-          lattice_v, GridDefaultSimd(4, vector_type::Nsimd()),
+          lattice_v, 
+          GridDefaultSimd(4, vector_type::Nsimd()),
           mpi_v));
       set_rb(SpaceTimeGrid::makeFourDimRedBlackGrid(grid_.get()));
-    } else {
-      std::cout << GridLogError 
-          << "Error in GridFourDimModule: lattice dimension different from 4"
-          << std::endl;
+    }
+    else
+    {
+      std::cout << GridLogError
+                << "Error in GridFourDimModule: lattice dimension different from 4"
+                << std::endl;
       exit(1);
     }
   }
diff --git a/lib/util/Init.cc b/lib/util/Init.cc
index fe3b1734..35a569ba 100644
--- a/lib/util/Init.cc
+++ b/lib/util/Init.cc
@@ -377,7 +377,7 @@ void Grid_init(int *argc,char ***argv)
   std::cout << GridLogDebug << "Requesting "<< CartesianCommunicator::MAX_MPI_SHM_BYTES <<" byte stencil comms buffers "<<std::endl;
 
   if( GridCmdOptionExists(*argv,*argv+*argc,"--decomposition") ){
-    std::cout<<GridLogMessage<<"Grid Decomposition\n";
+    std::cout<<GridLogMessage<<"Grid Default Decomposition patterns\n";
     std::cout<<GridLogMessage<<"\tOpenMP threads : "<<GridThread::GetThreads()<<std::endl;
     std::cout<<GridLogMessage<<"\tMPI tasks      : "<<GridCmdVectorIntToString(GridDefaultMpi())<<std::endl;
     std::cout<<GridLogMessage<<"\tvRealF         : "<<sizeof(vRealF)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealF::Nsimd()))<<std::endl;
diff --git a/tests/hmc/Test_hmc_EOMobiusRatio.cc b/tests/hmc/Test_hmc_EOMobiusRatio.cc
index 4b5da356..4b4555e3 100644
--- a/tests/hmc/Test_hmc_EOMobiusRatio.cc
+++ b/tests/hmc/Test_hmc_EOMobiusRatio.cc
@@ -40,12 +40,6 @@ namespace Grid{
 				    double, StoppingCondition,
 				    int, MaxCGIterations,
 				    bool, ApplySmearing);
-
-    //template <class ReaderClass >
-    //FermionParameters(Reader<ReaderClass>& Reader){
-    //  read(Reader, "Mobius", *this);
-    //}
-
   };
 
   
@@ -113,9 +107,12 @@ int main(int argc, char **argv) {
   bool ApplySmearing = MyParams.Mobius.ApplySmearing;
   
   
+  // Use this if you want to tweak the default decomposition
+  std::vector<int> simd_lanes({2,2,1,1});
 
-  // Grid from the command line
-  TheHMC.Resources.AddFourDimGrid("gauge");
+  // Grid from the command line arguments --grid and --mpi
+  // drop the simd_lanes argument to fall back to the default decomposition for the SIMD lanes
+  TheHMC.Resources.AddFourDimGrid("gauge", simd_lanes);
   // Possibile to create the module by hand 
   // hardcoding parameters or using a Reader
 

From dbe4d7850c1e132f538e4aead7869ba703a21ec5 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Sun, 6 Aug 2017 10:49:45 +0100
Subject: [PATCH 028/115] Make a test file compatible with all architectures

---
 tests/hmc/Test_hmc_EOMobiusRatio.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/hmc/Test_hmc_EOMobiusRatio.cc b/tests/hmc/Test_hmc_EOMobiusRatio.cc
index 4b4555e3..d6ca96db 100644
--- a/tests/hmc/Test_hmc_EOMobiusRatio.cc
+++ b/tests/hmc/Test_hmc_EOMobiusRatio.cc
@@ -108,11 +108,16 @@ int main(int argc, char **argv) {
   
   
   // Use this if you want to tweak the default decomposition
-  std::vector<int> simd_lanes({2,2,1,1});
+  // commented out as very architecture speficic
+  
+  //std::vector<int> simd_lanes({2,2,1,1});
 
   // Grid from the command line arguments --grid and --mpi
   // drop the simd_lanes argument to fall back to the default decomposition for the SIMD lanes
-  TheHMC.Resources.AddFourDimGrid("gauge", simd_lanes);
+  
+  //TheHMC.Resources.AddFourDimGrid("gauge", simd_lanes); // tweak the SIMD lanes
+  TheHMC.Resources.AddFourDimGrid("gauge"); // use default simd lanes decomposition
+  
   // Possibile to create the module by hand 
   // hardcoding parameters or using a Reader
 

From 06e6f8de00528ede75f248f98d48eca715d79630 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Tue, 8 Aug 2017 10:22:12 +0100
Subject: [PATCH 029/115] Check that the reduced dim is an integer

---
 lib/cartesian/Cartesian_red_black.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h
index 3037de00..e58999c5 100644
--- a/lib/cartesian/Cartesian_red_black.h
+++ b/lib/cartesian/Cartesian_red_black.h
@@ -176,7 +176,8 @@ public:
 
 	// Use a reduced simd grid
 	_simd_layout[d] = simd_layout[d];
-	_rdimensions[d]= _ldimensions[d]/_simd_layout[d];
+	_rdimensions[d]= _ldimensions[d]/_simd_layout[d]; // this is not checking if this is integer
+  assert(_rdimensions[d]*_simd_layout[d] == _ldimensions[d]);
 	assert(_rdimensions[d]>0);
 
 	// all elements of a simd vector must have same checkerboard.

From 44051aecd1eb0abc7a61ac814654491804455347 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Tue, 8 Aug 2017 10:31:12 +0100
Subject: [PATCH 030/115] Checking for integer divisions in cartesian full

---
 lib/cartesian/Cartesian_full.h | 130 +++++++++++++++++----------------
 1 file changed, 67 insertions(+), 63 deletions(-)

diff --git a/lib/cartesian/Cartesian_full.h b/lib/cartesian/Cartesian_full.h
index b0e47fa4..815e3b22 100644
--- a/lib/cartesian/Cartesian_full.h
+++ b/lib/cartesian/Cartesian_full.h
@@ -62,77 +62,81 @@ public:
       return shift;
     }
     GridCartesian(const std::vector<int> &dimensions,
-		  const std::vector<int> &simd_layout,
-		  const std::vector<int> &processor_grid
-		  ) : GridBase(processor_grid)
+                  const std::vector<int> &simd_layout,
+                  const std::vector<int> &processor_grid) : GridBase(processor_grid)
     {
-        ///////////////////////
-        // Grid information
-        ///////////////////////
-        _ndimension = dimensions.size();
-            
-        _fdimensions.resize(_ndimension);
-        _gdimensions.resize(_ndimension);
-        _ldimensions.resize(_ndimension);
-        _rdimensions.resize(_ndimension);
-        _simd_layout.resize(_ndimension);
-	_lstart.resize(_ndimension);
-	_lend.resize(_ndimension);
-            
-        _ostride.resize(_ndimension);
-        _istride.resize(_ndimension);
-            
-        _fsites = _gsites = _osites = _isites = 1;
+      ///////////////////////
+      // Grid information
+      ///////////////////////
+      _ndimension = dimensions.size();
 
-        for(int d=0;d<_ndimension;d++){
-	  _fdimensions[d] = dimensions[d]; // Global dimensions
-	  _gdimensions[d] = _fdimensions[d]; // Global dimensions
-	  _simd_layout[d] = simd_layout[d];
-	  _fsites = _fsites * _fdimensions[d];
-	  _gsites = _gsites * _gdimensions[d];
+      _fdimensions.resize(_ndimension);
+      _gdimensions.resize(_ndimension);
+      _ldimensions.resize(_ndimension);
+      _rdimensions.resize(_ndimension);
+      _simd_layout.resize(_ndimension);
+      _lstart.resize(_ndimension);
+      _lend.resize(_ndimension);
 
-	  //FIXME check for exact division
+      _ostride.resize(_ndimension);
+      _istride.resize(_ndimension);
 
-	  // Use a reduced simd grid
-	  _ldimensions[d]= _gdimensions[d]/_processors[d];  //local dimensions
-	  _rdimensions[d]= _ldimensions[d]/_simd_layout[d]; //overdecomposition
-	  _lstart[d]     = _processor_coor[d]*_ldimensions[d];
-	  _lend[d]       = _processor_coor[d]*_ldimensions[d]+_ldimensions[d]-1;
-	  _osites  *= _rdimensions[d];
-	  _isites  *= _simd_layout[d];
-                
-	  // Addressing support
-	  if ( d==0 ) {
-	    _ostride[d] = 1;
-	    _istride[d] = 1;
-	  } else {
-	    _ostride[d] = _ostride[d-1]*_rdimensions[d-1];
-	    _istride[d] = _istride[d-1]*_simd_layout[d-1];
-	  }
+      _fsites = _gsites = _osites = _isites = 1;
+
+      for (int d = 0; d < _ndimension; d++)
+      {
+        _fdimensions[d] = dimensions[d];   // Global dimensions
+        _gdimensions[d] = _fdimensions[d]; // Global dimensions
+        _simd_layout[d] = simd_layout[d];
+        _fsites = _fsites * _fdimensions[d];
+        _gsites = _gsites * _gdimensions[d];
+
+        // Use a reduced simd grid
+        _ldimensions[d] = _gdimensions[d] / _processors[d]; //local dimensions
+        assert(_ldimensions[d] * _processors[d] == _gdimensions[d]);
+
+        _rdimensions[d] = _ldimensions[d] / _simd_layout[d]; //overdecomposition
+        assert(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
+
+        _lstart[d] = _processor_coor[d] * _ldimensions[d];
+        _lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1;
+        _osites *= _rdimensions[d];
+        _isites *= _simd_layout[d];
+
+        // Addressing support
+        if (d == 0)
+        {
+          _ostride[d] = 1;
+          _istride[d] = 1;
         }
-        
-        ///////////////////////
-        // subplane information
-        ///////////////////////
-        _slice_block.resize(_ndimension);
-        _slice_stride.resize(_ndimension);
-        _slice_nblock.resize(_ndimension);
-            
-        int block =1;
-        int nblock=1;
-        for(int d=0;d<_ndimension;d++) nblock*=_rdimensions[d];
-            
-        for(int d=0;d<_ndimension;d++){
-            nblock/=_rdimensions[d];
-            _slice_block[d] =block;
-            _slice_stride[d]=_ostride[d]*_rdimensions[d];
-            _slice_nblock[d]=nblock;
-            block = block*_rdimensions[d];
+        else
+        {
+          _ostride[d] = _ostride[d - 1] * _rdimensions[d - 1];
+          _istride[d] = _istride[d - 1] * _simd_layout[d - 1];
         }
+      }
 
+      ///////////////////////
+      // subplane information
+      ///////////////////////
+      _slice_block.resize(_ndimension);
+      _slice_stride.resize(_ndimension);
+      _slice_nblock.resize(_ndimension);
+
+      int block = 1;
+      int nblock = 1;
+      for (int d = 0; d < _ndimension; d++)
+        nblock *= _rdimensions[d];
+
+      for (int d = 0; d < _ndimension; d++)
+      {
+        nblock /= _rdimensions[d];
+        _slice_block[d] = block;
+        _slice_stride[d] = _ostride[d] * _rdimensions[d];
+        _slice_nblock[d] = nblock;
+        block = block * _rdimensions[d];
+      }
     };
 };
-
-
 }
 #endif

From 8a3fe60a27e4573faca940efd33d18a7d468c764 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Tue, 8 Aug 2017 11:36:20 +0100
Subject: [PATCH 031/115] Added more  asserts at grid creation time

---
 lib/cartesian/Cartesian_red_black.h | 192 +++++++++++++++-------------
 1 file changed, 105 insertions(+), 87 deletions(-)

diff --git a/lib/cartesian/Cartesian_red_black.h b/lib/cartesian/Cartesian_red_black.h
index e58999c5..b1a5b9ef 100644
--- a/lib/cartesian/Cartesian_red_black.h
+++ b/lib/cartesian/Cartesian_red_black.h
@@ -131,21 +131,21 @@ public:
       Init(dimensions,simd_layout,processor_grid,checker_dim_mask,0);
     }
     void Init(const std::vector<int> &dimensions,
-	      const std::vector<int> &simd_layout,
-	      const std::vector<int> &processor_grid,
-	      const std::vector<int> &checker_dim_mask,
-	      int checker_dim)
+              const std::vector<int> &simd_layout,
+              const std::vector<int> &processor_grid,
+              const std::vector<int> &checker_dim_mask,
+              int checker_dim)
     {
-    ///////////////////////
-    // Grid information
-    ///////////////////////
+      ///////////////////////
+      // Grid information
+      ///////////////////////
       _checker_dim = checker_dim;
-      assert(checker_dim_mask[checker_dim]==1);
+      assert(checker_dim_mask[checker_dim] == 1);
       _ndimension = dimensions.size();
-      assert(checker_dim_mask.size()==_ndimension);
-      assert(processor_grid.size()==_ndimension);
-      assert(simd_layout.size()==_ndimension);
-      
+      assert(checker_dim_mask.size() == _ndimension);
+      assert(processor_grid.size() == _ndimension);
+      assert(simd_layout.size() == _ndimension);
+
       _fdimensions.resize(_ndimension);
       _gdimensions.resize(_ndimension);
       _ldimensions.resize(_ndimension);
@@ -153,115 +153,133 @@ public:
       _simd_layout.resize(_ndimension);
       _lstart.resize(_ndimension);
       _lend.resize(_ndimension);
-      
+
       _ostride.resize(_ndimension);
       _istride.resize(_ndimension);
-      
+
       _fsites = _gsites = _osites = _isites = 1;
-	
-      _checker_dim_mask=checker_dim_mask;
 
-      for(int d=0;d<_ndimension;d++){
-	_fdimensions[d] = dimensions[d];
-	_gdimensions[d] = _fdimensions[d];
-	_fsites = _fsites * _fdimensions[d];
-	_gsites = _gsites * _gdimensions[d];
-        
-	if (d==_checker_dim) {
-	  _gdimensions[d] = _gdimensions[d]/2; // Remove a checkerboard
-	}
-	_ldimensions[d] = _gdimensions[d]/_processors[d];
-	_lstart[d]     = _processor_coor[d]*_ldimensions[d];
-	_lend[d]       = _processor_coor[d]*_ldimensions[d]+_ldimensions[d]-1;
+      _checker_dim_mask = checker_dim_mask;
 
-	// Use a reduced simd grid
-	_simd_layout[d] = simd_layout[d];
-	_rdimensions[d]= _ldimensions[d]/_simd_layout[d]; // this is not checking if this is integer
-  assert(_rdimensions[d]*_simd_layout[d] == _ldimensions[d]);
-	assert(_rdimensions[d]>0);
+      for (int d = 0; d < _ndimension; d++)
+      {
+        _fdimensions[d] = dimensions[d];
+        _gdimensions[d] = _fdimensions[d];
+        _fsites = _fsites * _fdimensions[d];
+        _gsites = _gsites * _gdimensions[d];
 
-	// all elements of a simd vector must have same checkerboard.
-	// If Ls vectorised, this must still be the case; e.g. dwf rb5d
-	if ( _simd_layout[d]>1 ) {
-	  if ( checker_dim_mask[d] ) { 
-	    assert( (_rdimensions[d]&0x1) == 0 );
-	  }
-	}
+        if (d == _checker_dim)
+        {
+          assert((_gdimensions[d] & 0x1) == 0);
+          _gdimensions[d] = _gdimensions[d] / 2; // Remove a checkerboard
+        }
+        _ldimensions[d] = _gdimensions[d] / _processors[d];
+        assert(_ldimensions[d] * _processors[d] == _gdimensions[d]);
+        _lstart[d] = _processor_coor[d] * _ldimensions[d];
+        _lend[d] = _processor_coor[d] * _ldimensions[d] + _ldimensions[d] - 1;
 
-	_osites *= _rdimensions[d];
-	_isites *= _simd_layout[d];
-        
-	// Addressing support
-	if ( d==0 ) {
-	  _ostride[d] = 1;
-	  _istride[d] = 1;
-	} else {
-	  _ostride[d] = _ostride[d-1]*_rdimensions[d-1];
-	  _istride[d] = _istride[d-1]*_simd_layout[d-1];
-	}
+        // Use a reduced simd grid
+        _simd_layout[d] = simd_layout[d];
+        _rdimensions[d] = _ldimensions[d] / _simd_layout[d]; // this is not checking if this is integer
+        assert(_rdimensions[d] * _simd_layout[d] == _ldimensions[d]);
+        assert(_rdimensions[d] > 0);
 
+        // all elements of a simd vector must have same checkerboard.
+        // If Ls vectorised, this must still be the case; e.g. dwf rb5d
+        if (_simd_layout[d] > 1)
+        {
+          if (checker_dim_mask[d])
+          {
+            assert((_rdimensions[d] & 0x1) == 0);
+          }
+        }
 
+        _osites *= _rdimensions[d];
+        _isites *= _simd_layout[d];
+
+        // Addressing support
+        if (d == 0)
+        {
+          _ostride[d] = 1;
+          _istride[d] = 1;
+        }
+        else
+        {
+          _ostride[d] = _ostride[d - 1] * _rdimensions[d - 1];
+          _istride[d] = _istride[d - 1] * _simd_layout[d - 1];
+        }
       }
-            
+
       ////////////////////////////////////////////////////////////////////////////////////////////
       // subplane information
       ////////////////////////////////////////////////////////////////////////////////////////////
       _slice_block.resize(_ndimension);
       _slice_stride.resize(_ndimension);
       _slice_nblock.resize(_ndimension);
-        
-      int block =1;
-      int nblock=1;
-      for(int d=0;d<_ndimension;d++) nblock*=_rdimensions[d];
-      
-      for(int d=0;d<_ndimension;d++){
-	nblock/=_rdimensions[d];
-	_slice_block[d] =block;
-	_slice_stride[d]=_ostride[d]*_rdimensions[d];
-	_slice_nblock[d]=nblock;
-	block = block*_rdimensions[d];
+
+      int block = 1;
+      int nblock = 1;
+      for (int d = 0; d < _ndimension; d++)
+        nblock *= _rdimensions[d];
+
+      for (int d = 0; d < _ndimension; d++)
+      {
+        nblock /= _rdimensions[d];
+        _slice_block[d] = block;
+        _slice_stride[d] = _ostride[d] * _rdimensions[d];
+        _slice_nblock[d] = nblock;
+        block = block * _rdimensions[d];
       }
 
       ////////////////////////////////////////////////
       // Create a checkerboard lookup table
       ////////////////////////////////////////////////
       int rvol = 1;
-      for(int d=0;d<_ndimension;d++){
-	rvol=rvol * _rdimensions[d];
+      for (int d = 0; d < _ndimension; d++)
+      {
+        rvol = rvol * _rdimensions[d];
       }
       _checker_board.resize(rvol);
-      for(int osite=0;osite<_osites;osite++){
-	_checker_board[osite] = CheckerBoardFromOindex (osite);
+      for (int osite = 0; osite < _osites; osite++)
+      {
+        _checker_board[osite] = CheckerBoardFromOindex(osite);
       }
-      
     };
-protected:
+
+  protected:
     virtual int oIndex(std::vector<int> &coor)
     {
-      int idx=0;
-      for(int d=0;d<_ndimension;d++) {
-	if( d==_checker_dim ) {
-	  idx+=_ostride[d]*((coor[d]/2)%_rdimensions[d]);
-	} else {
-	  idx+=_ostride[d]*(coor[d]%_rdimensions[d]);
-	}
+      int idx = 0;
+      for (int d = 0; d < _ndimension; d++)
+      {
+        if (d == _checker_dim)
+        {
+          idx += _ostride[d] * ((coor[d] / 2) % _rdimensions[d]);
+        }
+        else
+        {
+          idx += _ostride[d] * (coor[d] % _rdimensions[d]);
+        }
       }
       return idx;
     };
-        
+
     virtual int iIndex(std::vector<int> &lcoor)
     {
-        int idx=0;
-        for(int d=0;d<_ndimension;d++) {
-	  if( d==_checker_dim ) {
-	    idx+=_istride[d]*(lcoor[d]/(2*_rdimensions[d]));
-	  } else { 
-	    idx+=_istride[d]*(lcoor[d]/_rdimensions[d]);
-	  }
-	}
-        return idx;
+      int idx = 0;
+      for (int d = 0; d < _ndimension; d++)
+      {
+        if (d == _checker_dim)
+        {
+          idx += _istride[d] * (lcoor[d] / (2 * _rdimensions[d]));
+        }
+        else
+        {
+          idx += _istride[d] * (lcoor[d] / _rdimensions[d]);
+        }
+      }
+      return idx;
     }
 };
-
 }
 #endif

From 202a7fe900461bf4f65947abc637b6c679839c15 Mon Sep 17 00:00:00 2001
From: David Murphy <david.murphy@ligo.org>
Date: Tue, 15 Aug 2017 13:36:08 -0400
Subject: [PATCH 032/115] Re-import DWF and abstract base EOFA fermion classes
 and tests

---
 lib/qcd/action/fermion/AbstractEOFAFermion.h  | 100 +++
 .../action/fermion/DomainWallEOFAFermion.cc   | 440 +++++++++++++
 .../action/fermion/DomainWallEOFAFermion.h    | 115 ++++
 .../fermion/DomainWallEOFAFermioncache.cc     | 248 +++++++
 .../fermion/DomainWallEOFAFermiondense.cc     | 159 +++++
 .../fermion/DomainWallEOFAFermionssp.cc       | 168 +++++
 .../fermion/DomainWallEOFAFermionvec.cc       | 605 ++++++++++++++++++
 lib/qcd/action/fermion/Fermion.h              |  26 +-
 lib/qcd/modules/FermionOperatorModules.h      |  49 +-
 tests/core/Test_dwf_eofa_even_odd.cc          | 239 +++++++
 tests/debug/Test_reweight_dwf_eofa.cc         | 206 ++++++
 11 files changed, 2346 insertions(+), 9 deletions(-)
 create mode 100644 lib/qcd/action/fermion/AbstractEOFAFermion.h
 create mode 100644 lib/qcd/action/fermion/DomainWallEOFAFermion.cc
 create mode 100644 lib/qcd/action/fermion/DomainWallEOFAFermion.h
 create mode 100644 lib/qcd/action/fermion/DomainWallEOFAFermioncache.cc
 create mode 100644 lib/qcd/action/fermion/DomainWallEOFAFermiondense.cc
 create mode 100644 lib/qcd/action/fermion/DomainWallEOFAFermionssp.cc
 create mode 100644 lib/qcd/action/fermion/DomainWallEOFAFermionvec.cc
 create mode 100644 tests/core/Test_dwf_eofa_even_odd.cc
 create mode 100644 tests/debug/Test_reweight_dwf_eofa.cc

diff --git a/lib/qcd/action/fermion/AbstractEOFAFermion.h b/lib/qcd/action/fermion/AbstractEOFAFermion.h
new file mode 100644
index 00000000..abe06b8c
--- /dev/null
+++ b/lib/qcd/action/fermion/AbstractEOFAFermion.h
@@ -0,0 +1,100 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/AbstractEOFAFermion.h
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef  GRID_QCD_ABSTRACT_EOFA_FERMION_H
+#define  GRID_QCD_ABSTRACT_EOFA_FERMION_H
+
+#include <Grid/qcd/action/fermion/CayleyFermion5D.h>
+
+namespace Grid {
+namespace QCD {
+
+    // DJM: Abstract base class for EOFA fermion types.
+    // Defines layout of additional EOFA-specific parameters and operators.
+    // Use to construct EOFA pseudofermion actions that are agnostic to Shamir / Mobius / etc.,
+    // and ensure that no one can construct EOFA pseudofermion action with non-EOFA fermion type.
+    template<class Impl>
+    class AbstractEOFAFermion : public CayleyFermion5D<Impl> {
+        public:
+            INHERIT_IMPL_TYPES(Impl);
+
+        public:
+            // Fermion operator: D(mq1) + shift*\gamma_{5}*R_{5}*\Delta_{\pm}(mq2,mq3)*P_{\pm}
+            RealD mq1;
+            RealD mq2;
+            RealD mq3;
+            RealD shift;
+            int pm;
+
+            RealD alpha; // Mobius scale
+            RealD k;     // EOFA normalization constant
+
+            virtual void Instantiatable(void) = 0;
+
+            // EOFA-specific operations
+            // Force user to implement in derived classes
+            virtual void Omega    (const FermionField& in, FermionField& out, int sign, int dag) = 0;
+            virtual void Dtilde   (const FermionField& in, FermionField& out) = 0;
+            virtual void DtildeInv(const FermionField& in, FermionField& out) = 0;
+
+            // Implement derivatives in base clcass: for EOFA both DWF and Mobius just need d(Dw)/dU
+            virtual void MDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag){
+                this->DhopDeriv(mat, U, V, dag);
+            };
+            virtual void MoeDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag){
+                this->DhopDerivOE(mat, U, V, dag);
+            };
+            virtual void MeoDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag){
+                this->DhopDerivEO(mat, U, V, dag);
+            };
+
+            // Recompute 5D coefficients for different value of shift constant (needed for heatbath loop over poles)
+            virtual void RefreshShiftCoefficients(RealD new_shift) = 0;
+
+            // Constructors
+            AbstractEOFAFermion(GaugeField&            _Umu,
+                                GridCartesian&         FiveDimGrid,
+                                GridRedBlackCartesian& FiveDimRedBlackGrid,
+                                GridCartesian&         FourDimGrid,
+                                GridRedBlackCartesian& FourDimRedBlackGrid,
+                                RealD _mq1, RealD _mq2, RealD _mq3, RealD _shift, int _pm,
+                                RealD _M5, RealD _b, RealD _c, const ImplParams &p= ImplParams()) :
+                                CayleyFermion5D<Impl>(_Umu, FiveDimGrid, FiveDimRedBlackGrid, FourDimGrid, FourDimRedBlackGrid, _mq1, _M5, p),
+                                mq1(_mq1), mq2(_mq2), mq3(_mq3), shift(_shift), pm(_pm)
+            {
+                int Ls = this->Ls;
+                this->alpha = _b + _c;
+                this->k = this->alpha * (_mq3 - _mq2) * std::pow(this->alpha+1.0,2*Ls) /
+                    ( std::pow(alpha+1.0,Ls) + _mq2*std::pow(alpha-1.0,Ls) ) /
+                    ( std::pow(alpha+1.0,Ls) + _mq3*std::pow(alpha-1.0,Ls) );
+            }
+    };
+}}
+
+#endif
diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermion.cc b/lib/qcd/action/fermion/DomainWallEOFAFermion.cc
new file mode 100644
index 00000000..dc4f6504
--- /dev/null
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermion.cc
@@ -0,0 +1,440 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/DomainWallEOFAFermion.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid_Eigen_Dense.h>
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/DomainWallEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+    template<class Impl>
+    DomainWallEOFAFermion<Impl>::DomainWallEOFAFermion(
+      GaugeField            &_Umu,
+      GridCartesian         &FiveDimGrid,
+      GridRedBlackCartesian &FiveDimRedBlackGrid,
+      GridCartesian         &FourDimGrid,
+      GridRedBlackCartesian &FourDimRedBlackGrid,
+      RealD _mq1, RealD _mq2, RealD _mq3,
+      RealD _shift, int _pm, RealD _M5, const ImplParams &p) :
+    AbstractEOFAFermion<Impl>(_Umu, FiveDimGrid, FiveDimRedBlackGrid,
+        FourDimGrid, FourDimRedBlackGrid, _mq1, _mq2, _mq3,
+        _shift, _pm, _M5, 1.0, 0.0, p)
+    {
+        RealD eps = 1.0;
+        Approx::zolotarev_data *zdata = Approx::higham(eps,this->Ls);
+        assert(zdata->n == this->Ls);
+
+        std::cout << GridLogMessage << "DomainWallEOFAFermion with Ls=" << this->Ls << std::endl;
+        this->SetCoefficientsTanh(zdata, 1.0, 0.0);
+
+        Approx::zolotarev_free(zdata);
+    }
+
+    /***************************************************************
+    /* Additional EOFA operators only called outside the inverter.
+    /* Since speed is not essential, simple axpby-style
+    /* implementations should be fine.
+    /***************************************************************/
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::Omega(const FermionField& psi, FermionField& Din, int sign, int dag)
+    {
+        int Ls = this->Ls;
+
+        Din = zero;
+        if((sign == 1) && (dag == 0)){ axpby_ssp(Din, 0.0, psi, 1.0, psi, Ls-1, 0); }
+        else if((sign == -1) && (dag == 0)){ axpby_ssp(Din, 0.0, psi, 1.0, psi, 0, 0); }
+        else if((sign == 1 ) && (dag == 1)){ axpby_ssp(Din, 0.0, psi, 1.0, psi, 0, Ls-1); }
+        else if((sign == -1) && (dag == 1)){ axpby_ssp(Din, 0.0, psi, 1.0, psi, 0, 0); }
+    }
+
+    // This is just the identity for DWF
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::Dtilde(const FermionField& psi, FermionField& chi){ chi = psi; }
+
+    // This is just the identity for DWF
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::DtildeInv(const FermionField& psi, FermionField& chi){ chi = psi; }
+
+    /*****************************************************************************************************/
+
+    template<class Impl>
+    RealD DomainWallEOFAFermion<Impl>::M(const FermionField& psi, FermionField& chi)
+    {
+        int Ls = this->Ls;
+
+        FermionField Din(psi._grid);
+
+        this->Meooe5D(psi, Din);
+        this->DW(Din, chi, DaggerNo);
+        axpby(chi, 1.0, 1.0, chi, psi);
+        this->M5D(psi, chi);
+        return(norm2(chi));
+    }
+
+    template<class Impl>
+    RealD DomainWallEOFAFermion<Impl>::Mdag(const FermionField& psi, FermionField& chi)
+    {
+        int Ls = this->Ls;
+
+        FermionField Din(psi._grid);
+
+        this->DW(psi, Din, DaggerYes);
+        this->MeooeDag5D(Din, chi);
+        this->M5Ddag(psi, chi);
+        axpby(chi, 1.0, 1.0, chi, psi);
+        return(norm2(chi));
+    }
+
+    /********************************************************************
+    /* Performance critical fermion operators called inside the inverter
+    /********************************************************************/
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::M5D(const FermionField& psi, FermionField& chi)
+    {
+        int   Ls    = this->Ls;
+        int   pm    = this->pm;
+        RealD shift = this->shift;
+        RealD mq1   = this->mq1;
+        RealD mq2   = this->mq2;
+        RealD mq3   = this->mq3;
+
+        // coefficients for shift operator ( = shift*\gamma_{5}*R_{5}*\Delta_{\pm}(mq2,mq3)*P_{\pm} )
+        Coeff_t shiftp(0.0), shiftm(0.0);
+        if(shift != 0.0){
+          if(pm == 1){ shiftp = shift*(mq3-mq2); }
+          else{ shiftm = -shift*(mq3-mq2); }
+        }
+
+        std::vector<Coeff_t> diag(Ls,1.0);
+        std::vector<Coeff_t> upper(Ls,-1.0); upper[Ls-1] = mq1 + shiftm;
+        std::vector<Coeff_t> lower(Ls,-1.0); lower[0]    = mq1 + shiftp;
+
+        #if(0)
+            std::cout << GridLogMessage << "DomainWallEOFAFermion::M5D(FF&,FF&):" << std::endl;
+            for(int i=0; i<diag.size(); ++i){
+                std::cout << GridLogMessage << "diag[" << i << "] =" << diag[i] << std::endl;
+            }
+            for(int i=0; i<upper.size(); ++i){
+                std::cout << GridLogMessage << "upper[" << i << "] =" << upper[i] << std::endl;
+            }
+            for(int i=0; i<lower.size(); ++i){
+                std::cout << GridLogMessage << "lower[" << i << "] =" << lower[i] << std::endl;
+            }
+        #endif
+
+        this->M5D(psi, chi, chi, lower, diag, upper);
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::M5Ddag(const FermionField& psi, FermionField& chi)
+    {
+        int   Ls    = this->Ls;
+        int   pm    = this->pm;
+        RealD shift = this->shift;
+        RealD mq1   = this->mq1;
+        RealD mq2   = this->mq2;
+        RealD mq3   = this->mq3;
+
+        // coefficients for shift operator ( = shift*\gamma_{5}*R_{5}*\Delta_{\pm}(mq2,mq3)*P_{\pm} )
+        Coeff_t shiftp(0.0), shiftm(0.0);
+        if(shift != 0.0){
+          if(pm == 1){ shiftp = shift*(mq3-mq2); }
+          else{ shiftm = -shift*(mq3-mq2); }
+        }
+
+        std::vector<Coeff_t> diag(Ls,1.0);
+        std::vector<Coeff_t> upper(Ls,-1.0); upper[Ls-1] = mq1 + shiftp;
+        std::vector<Coeff_t> lower(Ls,-1.0); lower[0]    = mq1 + shiftm;
+
+        #if(0)
+            std::cout << GridLogMessage << "DomainWallEOFAFermion::M5Ddag(FF&,FF&):" << std::endl;
+            for(int i=0; i<diag.size(); ++i){
+                std::cout << GridLogMessage << "diag[" << i << "] =" << diag[i] << std::endl;
+            }
+            for(int i=0; i<upper.size(); ++i){
+                std::cout << GridLogMessage << "upper[" << i << "] =" << upper[i] << std::endl;
+            }
+            for(int i=0; i<lower.size(); ++i){
+                std::cout << GridLogMessage << "lower[" << i << "] =" << lower[i] << std::endl;
+            }
+        #endif
+
+        this->M5Ddag(psi, chi, chi, lower, diag, upper);
+    }
+
+    // half checkerboard operations
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::Mooee(const FermionField& psi, FermionField& chi)
+    {
+        int Ls = this->Ls;
+
+        std::vector<Coeff_t> diag = this->bee;
+        std::vector<Coeff_t> upper(Ls);
+        std::vector<Coeff_t> lower(Ls);
+
+        for(int s=0; s<Ls; s++){
+          upper[s] = -this->cee[s];
+          lower[s] = -this->cee[s];
+        }
+        upper[Ls-1] = this->dm;
+        lower[0]    = this->dp;
+
+        this->M5D(psi, psi, chi, lower, diag, upper);
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeDag(const FermionField& psi, FermionField& chi)
+    {
+        int Ls = this->Ls;
+
+        std::vector<Coeff_t> diag = this->bee;
+        std::vector<Coeff_t> upper(Ls);
+        std::vector<Coeff_t> lower(Ls);
+
+        for(int s=0; s<Ls; s++){
+          upper[s] = -this->cee[s];
+          lower[s] = -this->cee[s];
+        }
+        upper[Ls-1] = this->dp;
+        lower[0]    = this->dm;
+
+        this->M5Ddag(psi, psi, chi, lower, diag, upper);
+    }
+
+    /****************************************************************************************/
+
+    //Zolo
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::SetCoefficientsInternal(RealD zolo_hi, std::vector<Coeff_t>& gamma, RealD b, RealD c)
+    {
+        int   Ls    = this->Ls;
+        int   pm    = this->pm;
+        RealD mq1   = this->mq1;
+        RealD mq2   = this->mq2;
+        RealD mq3   = this->mq3;
+        RealD shift = this->shift;
+
+        ////////////////////////////////////////////////////////
+        // Constants for the preconditioned matrix Cayley form
+        ////////////////////////////////////////////////////////
+        this->bs.resize(Ls);
+        this->cs.resize(Ls);
+        this->aee.resize(Ls);
+        this->aeo.resize(Ls);
+        this->bee.resize(Ls);
+        this->beo.resize(Ls);
+        this->cee.resize(Ls);
+        this->ceo.resize(Ls);
+
+        for(int i=0; i<Ls; ++i){
+          this->bee[i] = 4.0 - this->M5 + 1.0;
+          this->cee[i] = 1.0;
+        }
+
+        for(int i=0; i<Ls; ++i){
+          this->aee[i] = this->cee[i];
+          this->bs[i] = this->beo[i] = 1.0;
+          this->cs[i] = this->ceo[i] = 0.0;
+        }
+
+        //////////////////////////////////////////
+        // EOFA shift terms
+        //////////////////////////////////////////
+        if(pm == 1){
+          this->dp = mq1*this->cee[0] + shift*(mq3-mq2);
+          this->dm = mq1*this->cee[Ls-1];
+        } else if(this->pm == -1) {
+          this->dp = mq1*this->cee[0];
+          this->dm = mq1*this->cee[Ls-1] - shift*(mq3-mq2);
+        } else {
+          this->dp = mq1*this->cee[0];
+          this->dm = mq1*this->cee[Ls-1];
+        }
+
+        //////////////////////////////////////////
+        // LDU decomposition of eeoo
+        //////////////////////////////////////////
+        this->dee.resize(Ls+1);
+        this->lee.resize(Ls);
+        this->leem.resize(Ls);
+        this->uee.resize(Ls);
+        this->ueem.resize(Ls);
+
+        for(int i=0; i<Ls; ++i){
+
+          if(i < Ls-1){
+
+            this->lee[i] = -this->cee[i+1]/this->bee[i]; // sub-diag entry on the ith column
+
+            this->leem[i] = this->dm/this->bee[i];
+            for(int j=0; j<i; j++){ this->leem[i] *= this->aee[j]/this->bee[j]; }
+
+            this->dee[i] = this->bee[i];
+
+            this->uee[i] = -this->aee[i]/this->bee[i];   // up-diag entry on the ith row
+
+            this->ueem[i] = this->dp / this->bee[0];
+            for(int j=1; j<=i; j++){ this->ueem[i] *= this->cee[j]/this->bee[j]; }
+
+          } else {
+
+            this->lee[i]  = 0.0;
+            this->leem[i] = 0.0;
+            this->uee[i]  = 0.0;
+            this->ueem[i] = 0.0;
+
+          }
+        }
+
+        {
+          Coeff_t delta_d = 1.0 / this->bee[0];
+          for(int j=1; j<Ls-1; j++){ delta_d *= this->cee[j] / this->bee[j]; }
+          this->dee[Ls-1] = this->bee[Ls-1] + this->cee[0] * this->dm * delta_d;
+          this->dee[Ls] = this->bee[Ls-1] + this->cee[Ls-1] * this->dp * delta_d;
+        }
+
+        int inv = 1;
+        this->MooeeInternalCompute(0, inv, this->MatpInv, this->MatmInv);
+        this->MooeeInternalCompute(1, inv, this->MatpInvDag, this->MatmInvDag);
+    }
+
+    // Recompute Cayley-form coefficients for different shift
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::RefreshShiftCoefficients(RealD new_shift)
+    {
+        this->shift = new_shift;
+        Approx::zolotarev_data *zdata = Approx::higham(1.0, this->Ls);
+        this->SetCoefficientsTanh(zdata, 1.0, 0.0);
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInternalCompute(int dag, int inv,
+        Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm)
+    {
+        int Ls = this->Ls;
+
+        GridBase* grid = this->FermionRedBlackGrid();
+        int LLs = grid->_rdimensions[0];
+
+        if(LLs == Ls){
+            return; // Not vectorised in 5th direction
+        }
+
+        Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
+        Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);
+
+        for(int s=0; s<Ls; s++){
+            Pplus(s,s)  = this->bee[s];
+            Pminus(s,s) = this->bee[s];
+        }
+
+        for(int s=0; s<Ls-1; s++){
+            Pminus(s,s+1) = -this->cee[s];
+        }
+
+        for(int s=0; s<Ls-1; s++){
+            Pplus(s+1,s) = -this->cee[s+1];
+        }
+
+        Pplus (0,Ls-1) = this->dp;
+        Pminus(Ls-1,0) = this->dm;
+
+        Eigen::MatrixXcd PplusMat ;
+        Eigen::MatrixXcd PminusMat;
+
+        #if(0)
+            std::cout << GridLogMessage << "Pplus:" << std::endl;
+            for(int s=0; s<Ls; ++s){
+                for(int ss=0; ss<Ls; ++ss){
+                    std::cout << Pplus(s,ss) << "\t";
+                }
+                std::cout << std::endl;
+            }
+            std::cout << GridLogMessage << "Pminus:" << std::endl;
+            for(int s=0; s<Ls; ++s){
+                for(int ss=0; ss<Ls; ++ss){
+                    std::cout << Pminus(s,ss) << "\t";
+                }
+                std::cout << std::endl;
+            }
+        #endif
+
+        if(inv) {
+            PplusMat  = Pplus.inverse();
+            PminusMat = Pminus.inverse();
+        } else {
+            PplusMat  = Pplus;
+            PminusMat = Pminus;
+        }
+
+        if(dag){
+            PplusMat.adjointInPlace();
+            PminusMat.adjointInPlace();
+        }
+
+        typedef typename SiteHalfSpinor::scalar_type scalar_type;
+        const int Nsimd = Simd::Nsimd();
+        Matp.resize(Ls*LLs);
+        Matm.resize(Ls*LLs);
+
+        for(int s2=0; s2<Ls; s2++){
+        for(int s1=0; s1<LLs; s1++){
+            int istride = LLs;
+            int ostride = 1;
+            Simd Vp;
+            Simd Vm;
+            scalar_type *sp = (scalar_type*) &Vp;
+            scalar_type *sm = (scalar_type*) &Vm;
+            for(int l=0; l<Nsimd; l++){
+                if(switcheroo<Coeff_t>::iscomplex()) {
+                    sp[l] = PplusMat (l*istride+s1*ostride,s2);
+                    sm[l] = PminusMat(l*istride+s1*ostride,s2);
+                } else {
+                    // if real
+                    scalar_type tmp;
+                    tmp = PplusMat (l*istride+s1*ostride,s2);
+                    sp[l] = scalar_type(tmp.real(),tmp.real());
+                    tmp = PminusMat(l*istride+s1*ostride,s2);
+                    sm[l] = scalar_type(tmp.real(),tmp.real());
+                }
+            }
+            Matp[LLs*s2+s1] = Vp;
+            Matm[LLs*s2+s1] = Vm;
+        }}
+    }
+
+    FermOpTemplateInstantiate(DomainWallEOFAFermion);
+    GparityFermOpTemplateInstantiate(DomainWallEOFAFermion);
+
+}}
diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermion.h b/lib/qcd/action/fermion/DomainWallEOFAFermion.h
new file mode 100644
index 00000000..179736ba
--- /dev/null
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermion.h
@@ -0,0 +1,115 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/DomainWallEOFAFermion.h
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef  GRID_QCD_DOMAIN_WALL_EOFA_FERMION_H
+#define  GRID_QCD_DOMAIN_WALL_EOFA_FERMION_H
+
+#include <Grid/qcd/action/fermion/AbstractEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+    // DJM: EOFA with (Shamir) domain wall fermions.
+    // We overload and re-implement only the routines which have a different operator
+    // structure than the CayleyFermion5D base class.
+    template<class Impl>
+    class DomainWallEOFAFermion : public AbstractEOFAFermion<Impl>
+    {
+        public:
+          INHERIT_IMPL_TYPES(Impl);
+
+        public:
+          // Modified (0,Ls-1) and (Ls-1,0) elements of Mooee for red-black preconditioned Shamir EOFA
+          Coeff_t dm;
+          Coeff_t dp;
+
+          virtual void Instantiatable(void) {};
+
+          // EOFA specific operators
+          virtual void   Omega      (const FermionField& in, FermionField &out, int sign, int dag);
+          virtual void   Dtilde     (const FermionField& in, FermionField &out);
+          virtual void   DtildeInv  (const FermionField& in, FermionField &out);
+
+          // override multiply
+          virtual RealD  M          (const FermionField& in, FermionField& out);
+          virtual RealD  Mdag       (const FermionField& in, FermionField& out);
+
+          // half checkerboard operations
+          virtual void   Mooee      (const FermionField &in, FermionField &out);
+          virtual void   MooeeDag   (const FermionField &in, FermionField &out);
+          virtual void   MooeeInv   (const FermionField &in, FermionField &out);
+          virtual void   MooeeInvDag(const FermionField &in, FermionField &out);
+
+          virtual void   M5D        (const FermionField &psi, FermionField &chi);
+          virtual void   M5Ddag     (const FermionField &psi, FermionField &chi);
+
+          /////////////////////////////////////////////////////
+          // Instantiate different versions depending on Impl
+          /////////////////////////////////////////////////////
+          void M5D                 (const FermionField &psi, const FermionField &phi, FermionField &chi,
+                                        std::vector<Coeff_t> &lower, std::vector<Coeff_t> &diag, std::vector<Coeff_t> &upper);
+          void M5Ddag              (const FermionField &psi, const FermionField &phi, FermionField &chi,
+                                        std::vector<Coeff_t> &lower, std::vector<Coeff_t> &diag, std::vector<Coeff_t> &upper);
+          void MooeeInternal       (const FermionField &in, FermionField &out, int dag, int inv);
+          void MooeeInternalCompute(int dag, int inv, Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm);
+          void MooeeInternalAsm    (const FermionField &in, FermionField &out, int LLs, int site,
+                                        Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm);
+          void MooeeInternalZAsm   (const FermionField &in, FermionField &out, int LLs, int site,
+                                        Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm);
+
+          virtual void RefreshShiftCoefficients(RealD new_shift);
+
+          // Constructors
+          DomainWallEOFAFermion(GaugeField&            _Umu,
+                                GridCartesian&         FiveDimGrid,
+                                GridRedBlackCartesian& FiveDimRedBlackGrid,
+                                GridCartesian&         FourDimGrid,
+                                GridRedBlackCartesian& FourDimRedBlackGrid,
+                                RealD _mq1, RealD _mq2, RealD _mq3, RealD _shift,
+                                int pm, RealD _M5, const ImplParams& p=ImplParams());
+
+        protected:
+          virtual void SetCoefficientsInternal(RealD zolo_hi, std::vector<Coeff_t> &gamma, RealD b, RealD c);
+    };
+}}
+
+#define INSTANTIATE_DPERP_DWF_EOFA(A)\
+template void DomainWallEOFAFermion<A>::M5D(const FermionField& psi, const FermionField& phi, FermionField& chi,\
+    std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper); \
+template void DomainWallEOFAFermion<A>::M5Ddag(const FermionField& psi, const FermionField& phi, FermionField& chi,\
+    std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper); \
+template void DomainWallEOFAFermion<A>::MooeeInv(const FermionField& psi, FermionField& chi); \
+template void DomainWallEOFAFermion<A>::MooeeInvDag(const FermionField& psi, FermionField& chi);
+
+#undef  DOMAIN_WALL_EOFA_DPERP_DENSE
+#define DOMAIN_WALL_EOFA_DPERP_CACHE
+#undef  DOMAIN_WALL_EOFA_DPERP_LINALG
+#define DOMAIN_WALL_EOFA_DPERP_VEC
+
+#endif
diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermioncache.cc b/lib/qcd/action/fermion/DomainWallEOFAFermioncache.cc
new file mode 100644
index 00000000..0b214d31
--- /dev/null
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermioncache.cc
@@ -0,0 +1,248 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/DomainWallEOFAFermioncache.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/DomainWallEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+    // FIXME -- make a version of these routines with site loop outermost for cache reuse.
+
+    // Pminus fowards
+    // Pplus  backwards..
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::M5D(const FermionField& psi, const FermionField& phi,
+        FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+    {
+        int Ls = this->Ls;
+        GridBase* grid = psi._grid;
+
+        assert(phi.checkerboard == psi.checkerboard);
+        chi.checkerboard = psi.checkerboard;
+        // Flops = 6.0*(Nc*Ns) *Ls*vol
+        this->M5Dcalls++;
+        this->M5Dtime -= usecond();
+
+        parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){ // adds Ls
+            for(int s=0; s<Ls; s++){
+                auto tmp = psi._odata[0];
+                if(s==0) {
+                    spProj5m(tmp, psi._odata[ss+s+1]);
+                    chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+                    spProj5p(tmp, psi._odata[ss+Ls-1]);
+                    chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+                } else if(s==(Ls-1)) {
+                    spProj5m(tmp, psi._odata[ss+0]);
+                    chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+                    spProj5p(tmp, psi._odata[ss+s-1]);
+                    chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+                } else {
+                    spProj5m(tmp, psi._odata[ss+s+1]);
+                    chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+                    spProj5p(tmp, psi._odata[ss+s-1]);
+                    chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+                }
+            }
+        }
+
+        this->M5Dtime += usecond();
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::M5Ddag(const FermionField& psi, const FermionField& phi,
+        FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+    {
+        int Ls = this->Ls;
+        GridBase* grid = psi._grid;
+        assert(phi.checkerboard == psi.checkerboard);
+        chi.checkerboard=psi.checkerboard;
+
+        // Flops = 6.0*(Nc*Ns) *Ls*vol
+        this->M5Dcalls++;
+        this->M5Dtime -= usecond();
+
+        parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){ // adds Ls
+            auto tmp = psi._odata[0];
+            for(int s=0; s<Ls; s++){
+                if(s==0) {
+                    spProj5p(tmp, psi._odata[ss+s+1]);
+                    chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+                    spProj5m(tmp, psi._odata[ss+Ls-1]);
+                    chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+                } else if(s==(Ls-1)) {
+                    spProj5p(tmp, psi._odata[ss+0]);
+                    chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+                    spProj5m(tmp, psi._odata[ss+s-1]);
+                    chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+                } else {
+                    spProj5p(tmp, psi._odata[ss+s+1]);
+                    chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+                    spProj5m(tmp, psi._odata[ss+s-1]);
+                    chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+                }
+            }
+        }
+
+        this->M5Dtime += usecond();
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInv(const FermionField& psi, FermionField& chi)
+    {
+        GridBase* grid = psi._grid;
+        int Ls = this->Ls;
+
+        chi.checkerboard = psi.checkerboard;
+
+        this->MooeeInvCalls++;
+        this->MooeeInvTime -= usecond();
+
+        parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){ // adds Ls
+
+            auto tmp1 = psi._odata[0];
+            auto tmp2 = psi._odata[0];
+
+            // flops = 12*2*Ls + 12*2*Ls + 3*12*Ls + 12*2*Ls  = 12*Ls * (9) = 108*Ls flops
+            // Apply (L^{\prime})^{-1}
+            chi[ss] = psi[ss]; // chi[0]=psi[0]
+            for(int s=1; s<Ls; s++){
+                spProj5p(tmp1, chi[ss+s-1]);
+                chi[ss+s] = psi[ss+s] - this->lee[s-1]*tmp1;
+            }
+
+            // L_m^{-1}
+            for(int s=0; s<Ls-1; s++){ // Chi[ee] = 1 - sum[s<Ls-1] -leem[s]P_- chi
+                spProj5m(tmp1, chi[ss+s]);
+                chi[ss+Ls-1] = chi[ss+Ls-1] - this->leem[s]*tmp1;
+            }
+
+            // U_m^{-1} D^{-1}
+            for(int s=0; s<Ls-1; s++){ // Chi[s] + 1/d chi[s]
+                spProj5p(tmp1, chi[ss+Ls-1]);
+                chi[ss+s] = (1.0/this->dee[s])*chi[ss+s] - (this->ueem[s]/this->dee[Ls])*tmp1;
+            }
+            spProj5m(tmp2, chi[ss+Ls-1]);
+            chi[ss+Ls-1] = (1.0/this->dee[Ls])*tmp1 + (1.0/this->dee[Ls-1])*tmp2;
+
+            // Apply U^{-1}
+            for(int s=Ls-2; s>=0; s--){
+                spProj5m(tmp1, chi[ss+s+1]);
+                chi[ss+s] = chi[ss+s] - this->uee[s]*tmp1;
+            }
+        }
+
+        this->MooeeInvTime += usecond();
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInvDag(const FermionField& psi, FermionField& chi)
+    {
+        GridBase* grid = psi._grid;
+        int Ls = this->Ls;
+
+        assert(psi.checkerboard == psi.checkerboard);
+        chi.checkerboard = psi.checkerboard;
+
+        std::vector<Coeff_t> ueec(Ls);
+        std::vector<Coeff_t> deec(Ls+1);
+        std::vector<Coeff_t> leec(Ls);
+        std::vector<Coeff_t> ueemc(Ls);
+        std::vector<Coeff_t> leemc(Ls);
+
+        for(int s=0; s<ueec.size(); s++){
+            ueec[s]  = conjugate(this->uee[s]);
+            deec[s]  = conjugate(this->dee[s]);
+            leec[s]  = conjugate(this->lee[s]);
+            ueemc[s] = conjugate(this->ueem[s]);
+            leemc[s] = conjugate(this->leem[s]);
+        }
+        deec[Ls] = conjugate(this->dee[Ls]);
+
+        this->MooeeInvCalls++;
+        this->MooeeInvTime -= usecond();
+
+        parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){ // adds Ls
+
+            auto tmp1 = psi._odata[0];
+            auto tmp2 = psi._odata[0];
+
+            // Apply (U^{\prime})^{-dagger}
+            chi[ss] = psi[ss];
+            for(int s=1; s<Ls; s++){
+                spProj5m(tmp1, chi[ss+s-1]);
+                chi[ss+s] = psi[ss+s] - ueec[s-1]*tmp1;
+            }
+
+            // U_m^{-\dagger}
+            for(int s=0; s<Ls-1; s++){
+                spProj5p(tmp1, chi[ss+s]);
+                chi[ss+Ls-1] = chi[ss+Ls-1] - ueemc[s]*tmp1;
+            }
+
+            // L_m^{-\dagger} D^{-dagger}
+            for(int s=0; s<Ls-1; s++){
+                spProj5m(tmp1, chi[ss+Ls-1]);
+                chi[ss+s] = (1.0/deec[s])*chi[ss+s] - (leemc[s]/deec[Ls-1])*tmp1;
+            }
+            spProj5p(tmp2, chi[ss+Ls-1]);
+            chi[ss+Ls-1] = (1.0/deec[Ls-1])*tmp1 + (1.0/deec[Ls])*tmp2;
+
+            // Apply L^{-dagger}
+            for(int s=Ls-2; s>=0; s--){
+                spProj5p(tmp1, chi[ss+s+1]);
+                chi[ss+s] = chi[ss+s] - leec[s]*tmp1;
+            }
+        }
+
+        this->MooeeInvTime += usecond();
+    }
+
+    #ifdef DOMAIN_WALL_EOFA_DPERP_CACHE
+
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplD);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplD);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplD);
+
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplDF);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplDF);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplDF);
+
+    #endif
+
+}}
diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermiondense.cc b/lib/qcd/action/fermion/DomainWallEOFAFermiondense.cc
new file mode 100644
index 00000000..c27074d9
--- /dev/null
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermiondense.cc
@@ -0,0 +1,159 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/DomainWallEOFAFermiondense.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid_Eigen_Dense.h>
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/DomainWallEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+    /*
+    * Dense matrix versions of routines
+    */
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInvDag(const FermionField& psi, FermionField& chi)
+    {
+        this->MooeeInternal(psi, chi, DaggerYes, InverseYes);
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInv(const FermionField& psi, FermionField& chi)
+    {
+        this->MooeeInternal(psi, chi, DaggerNo, InverseYes);
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv)
+    {
+        int Ls = this->Ls;
+        int LLs = psi._grid->_rdimensions[0];
+        int vol = psi._grid->oSites()/LLs;
+
+        chi.checkerboard = psi.checkerboard;
+
+        assert(Ls==LLs);
+
+        Eigen::MatrixXd Pplus  = Eigen::MatrixXd::Zero(Ls,Ls);
+        Eigen::MatrixXd Pminus = Eigen::MatrixXd::Zero(Ls,Ls);
+
+        for(int s=0;s<Ls;s++){
+            Pplus(s,s)  = this->bee[s];
+            Pminus(s,s) = this->bee[s];
+        }
+
+        for(int s=0; s<Ls-1; s++){
+            Pminus(s,s+1) = -this->cee[s];
+        }
+
+        for(int s=0; s<Ls-1; s++){
+            Pplus(s+1,s) = -this->cee[s+1];
+        }
+
+        Pplus (0,Ls-1) = this->dp;
+        Pminus(Ls-1,0) = this->dm;
+
+        Eigen::MatrixXd PplusMat ;
+        Eigen::MatrixXd PminusMat;
+
+        if(inv) {
+            PplusMat  = Pplus.inverse();
+            PminusMat = Pminus.inverse();
+        } else {
+            PplusMat  = Pplus;
+            PminusMat = Pminus;
+        }
+
+        if(dag){
+            PplusMat.adjointInPlace();
+            PminusMat.adjointInPlace();
+        }
+
+        // For the non-vectorised s-direction this is simple
+
+        for(auto site=0; site<vol; site++){
+
+            SiteSpinor     SiteChi;
+            SiteHalfSpinor SitePplus;
+            SiteHalfSpinor SitePminus;
+
+            for(int s1=0; s1<Ls; s1++){
+                SiteChi = zero;
+                for(int s2=0; s2<Ls; s2++){
+                    int lex2 = s2 + Ls*site;
+                    if(PplusMat(s1,s2) != 0.0){
+                        spProj5p(SitePplus,psi[lex2]);
+                        accumRecon5p(SiteChi, PplusMat(s1,s2)*SitePplus);
+                    }
+                    if(PminusMat(s1,s2) != 0.0){
+                        spProj5m(SitePminus, psi[lex2]);
+                        accumRecon5m(SiteChi, PminusMat(s1,s2)*SitePminus);
+                    }
+                }
+                chi[s1+Ls*site] = SiteChi*0.5;
+            }
+        }
+    }
+
+    #ifdef DOMAIN_WALL_EOFA_DPERP_DENSE
+
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplD);
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplD);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplD);
+
+        template void DomainWallEOFAFermion<GparityWilsonImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<GparityWilsonImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<WilsonImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<WilsonImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<ZWilsonImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<ZWilsonImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplDF);
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplDF);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplDF);
+
+        template void DomainWallEOFAFermion<GparityWilsonImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<GparityWilsonImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<WilsonImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<WilsonImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<ZWilsonImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<ZWilsonImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+
+    #endif
+
+}}
diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermionssp.cc b/lib/qcd/action/fermion/DomainWallEOFAFermionssp.cc
new file mode 100644
index 00000000..80a4bf09
--- /dev/null
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermionssp.cc
@@ -0,0 +1,168 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/DomainWallEOFAFermionssp.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/DomainWallEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+    // FIXME -- make a version of these routines with site loop outermost for cache reuse.
+    // Pminus fowards
+    // Pplus  backwards
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::M5D(const FermionField& psi, const FermionField& phi,
+        FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+    {
+        Coeff_t one(1.0);
+        int Ls = this->Ls;
+        for(int s=0; s<Ls; s++){
+            if(s==0) {
+              axpby_ssp_pminus(chi, diag[s], phi, upper[s], psi, s, s+1);
+              axpby_ssp_pplus (chi, one, chi, lower[s], psi, s, Ls-1);
+            } else if (s==(Ls-1)) {
+              axpby_ssp_pminus(chi, diag[s], phi, upper[s], psi, s, 0);
+              axpby_ssp_pplus (chi, one, chi, lower[s], psi, s, s-1);
+            } else {
+              axpby_ssp_pminus(chi, diag[s], phi, upper[s], psi, s, s+1);
+              axpby_ssp_pplus(chi, one, chi, lower[s], psi, s, s-1);
+            }
+        }
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::M5Ddag(const FermionField& psi, const FermionField& phi,
+        FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+    {
+        Coeff_t one(1.0);
+        int Ls = this->Ls;
+        for(int s=0; s<Ls; s++){
+            if(s==0) {
+              axpby_ssp_pplus (chi, diag[s], phi, upper[s], psi, s, s+1);
+              axpby_ssp_pminus(chi, one, chi, lower[s], psi, s, Ls-1);
+            } else if (s==(Ls-1)) {
+              axpby_ssp_pplus (chi, diag[s], phi, upper[s], psi, s, 0);
+              axpby_ssp_pminus(chi, one, chi, lower[s], psi, s, s-1);
+            } else {
+              axpby_ssp_pplus (chi, diag[s], phi, upper[s], psi, s, s+1);
+              axpby_ssp_pminus(chi, one, chi, lower[s], psi, s, s-1);
+            }
+        }
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInv(const FermionField& psi, FermionField& chi)
+    {
+        Coeff_t one(1.0);
+        Coeff_t czero(0.0);
+        chi.checkerboard = psi.checkerboard;
+        int Ls = this->Ls;
+
+        FermionField tmp(psi._grid);
+
+        // Apply (L^{\prime})^{-1}
+        axpby_ssp(chi, one, psi, czero, psi, 0, 0);      // chi[0]=psi[0]
+        for(int s=1; s<Ls; s++){
+            axpby_ssp_pplus(chi, one, psi, -this->lee[s-1], chi, s, s-1);// recursion Psi[s] -lee P_+ chi[s-1]
+        }
+
+        // L_m^{-1}
+        for(int s=0; s<Ls-1; s++){ // Chi[ee] = 1 - sum[s<Ls-1] -leem[s]P_- chi
+            axpby_ssp_pminus(chi, one, chi, -this->leem[s], chi, Ls-1, s);
+        }
+
+        // U_m^{-1} D^{-1}
+        for(int s=0; s<Ls-1; s++){
+            axpby_ssp_pplus(chi, one/this->dee[s], chi, -this->ueem[s]/this->dee[Ls], chi, s, Ls-1);
+        }
+        axpby_ssp_pminus(tmp, czero, chi, one/this->dee[Ls-1], chi, Ls-1, Ls-1);
+        axpby_ssp_pplus(chi, one, tmp, one/this->dee[Ls], chi, Ls-1, Ls-1);
+
+        // Apply U^{-1}
+        for(int s=Ls-2; s>=0; s--){
+            axpby_ssp_pminus(chi, one, chi, -this->uee[s], chi, s, s+1);  // chi[Ls]
+        }
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInvDag(const FermionField& psi, FermionField& chi)
+    {
+        Coeff_t one(1.0);
+        Coeff_t czero(0.0);
+        chi.checkerboard = psi.checkerboard;
+        int Ls = this->Ls;
+
+        FermionField tmp(psi._grid);
+
+        // Apply (U^{\prime})^{-dagger}
+        axpby_ssp(chi, one, psi, czero, psi, 0, 0);      // chi[0]=psi[0]
+        for(int s=1; s<Ls; s++){
+            axpby_ssp_pminus(chi, one, psi, -conjugate(this->uee[s-1]), chi, s, s-1);
+        }
+
+        // U_m^{-\dagger}
+        for(int s=0; s<Ls-1; s++){
+            axpby_ssp_pplus(chi, one, chi, -conjugate(this->ueem[s]), chi, Ls-1, s);
+        }
+
+        // L_m^{-\dagger} D^{-dagger}
+        for(int s=0; s<Ls-1; s++){
+            axpby_ssp_pminus(chi, one/conjugate(this->dee[s]), chi, -conjugate(this->leem[s]/this->dee[Ls-1]), chi, s, Ls-1);
+        }
+        axpby_ssp_pminus(tmp, czero, chi, one/conjugate(this->dee[Ls-1]), chi, Ls-1, Ls-1);
+        axpby_ssp_pplus(chi, one, tmp, one/conjugate(this->dee[Ls]), chi, Ls-1, Ls-1);
+
+        // Apply L^{-dagger}
+        for(int s=Ls-2; s>=0; s--){
+            axpby_ssp_pplus(chi, one, chi, -conjugate(this->lee[s]), chi, s, s+1);  // chi[Ls]
+        }
+    }
+
+    #ifdef DOMAIN_WALL_EOFA_DPERP_LINALG
+
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplD);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplD);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplD);
+
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(WilsonImplDF);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(GparityWilsonImplDF);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(ZWilsonImplDF);
+
+    #endif
+
+}}
diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermionvec.cc b/lib/qcd/action/fermion/DomainWallEOFAFermionvec.cc
new file mode 100644
index 00000000..81ce448c
--- /dev/null
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermionvec.cc
@@ -0,0 +1,605 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/DomainWallEOFAFermionvec.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/DomainWallEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+    /*
+    * Dense matrix versions of routines
+    */
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInvDag(const FermionField& psi, FermionField& chi)
+    {
+        this->MooeeInternal(psi, chi, DaggerYes, InverseYes);
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInv(const FermionField& psi, FermionField& chi)
+    {
+        this->MooeeInternal(psi, chi, DaggerNo, InverseYes);
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::M5D(const FermionField& psi, const FermionField& phi,
+        FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+    {
+        GridBase* grid = psi._grid;
+        int Ls  = this->Ls;
+        int LLs = grid->_rdimensions[0];
+        const int nsimd = Simd::Nsimd();
+
+        Vector<iSinglet<Simd> > u(LLs);
+        Vector<iSinglet<Simd> > l(LLs);
+        Vector<iSinglet<Simd> > d(LLs);
+
+        assert(Ls/LLs == nsimd);
+        assert(phi.checkerboard == psi.checkerboard);
+
+        chi.checkerboard = psi.checkerboard;
+
+        // just directly address via type pun
+        typedef typename Simd::scalar_type scalar_type;
+        scalar_type* u_p = (scalar_type*) &u[0];
+        scalar_type* l_p = (scalar_type*) &l[0];
+        scalar_type* d_p = (scalar_type*) &d[0];
+
+        for(int o=0;o<LLs;o++){ // outer
+        for(int i=0;i<nsimd;i++){ //inner
+            int s  = o + i*LLs;
+            int ss = o*nsimd + i;
+            u_p[ss] = upper[s];
+            l_p[ss] = lower[s];
+            d_p[ss] = diag[s];
+        }}
+
+        this->M5Dcalls++;
+        this->M5Dtime -= usecond();
+
+        assert(Nc == 3);
+
+        parallel_for(int ss=0; ss<grid->oSites(); ss+=LLs){ // adds LLs
+
+            #if 0
+
+                alignas(64) SiteHalfSpinor hp;
+                alignas(64) SiteHalfSpinor hm;
+                alignas(64) SiteSpinor fp;
+                alignas(64) SiteSpinor fm;
+
+                for(int v=0; v<LLs; v++){
+
+                    int vp = (v+1)%LLs;
+                    int vm = (v+LLs-1)%LLs;
+
+                    spProj5m(hp, psi[ss+vp]);
+                    spProj5p(hm, psi[ss+vm]);
+
+                    if (vp <= v){ rotate(hp, hp, 1); }
+                    if (vm >= v){ rotate(hm, hm, nsimd-1); }
+
+                    hp = 0.5*hp;
+                    hm = 0.5*hm;
+
+                    spRecon5m(fp, hp);
+                    spRecon5p(fm, hm);
+
+                    chi[ss+v] = d[v]*phi[ss+v];
+                    chi[ss+v] = chi[ss+v] + u[v]*fp;
+                    chi[ss+v] = chi[ss+v] + l[v]*fm;
+
+                }
+
+            #else
+
+                for(int v=0; v<LLs; v++){
+
+                    vprefetch(psi[ss+v+LLs]);
+
+                    int vp = (v==LLs-1) ? 0     : v+1;
+                    int vm = (v==0)     ? LLs-1 : v-1;
+
+                    Simd hp_00 = psi[ss+vp]()(2)(0);
+                    Simd hp_01 = psi[ss+vp]()(2)(1);
+                    Simd hp_02 = psi[ss+vp]()(2)(2);
+                    Simd hp_10 = psi[ss+vp]()(3)(0);
+                    Simd hp_11 = psi[ss+vp]()(3)(1);
+                    Simd hp_12 = psi[ss+vp]()(3)(2);
+
+                    Simd hm_00 = psi[ss+vm]()(0)(0);
+                    Simd hm_01 = psi[ss+vm]()(0)(1);
+                    Simd hm_02 = psi[ss+vm]()(0)(2);
+                    Simd hm_10 = psi[ss+vm]()(1)(0);
+                    Simd hm_11 = psi[ss+vm]()(1)(1);
+                    Simd hm_12 = psi[ss+vm]()(1)(2);
+
+                    if(vp <= v){
+                        hp_00.v = Optimization::Rotate::tRotate<2>(hp_00.v);
+                        hp_01.v = Optimization::Rotate::tRotate<2>(hp_01.v);
+                        hp_02.v = Optimization::Rotate::tRotate<2>(hp_02.v);
+                        hp_10.v = Optimization::Rotate::tRotate<2>(hp_10.v);
+                        hp_11.v = Optimization::Rotate::tRotate<2>(hp_11.v);
+                        hp_12.v = Optimization::Rotate::tRotate<2>(hp_12.v);
+                    }
+
+                    if(vm >= v){
+                        hm_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_00.v);
+                        hm_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_01.v);
+                        hm_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_02.v);
+                        hm_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_10.v);
+                        hm_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_11.v);
+                        hm_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_12.v);
+                    }
+
+                    // Can force these to real arithmetic and save 2x.
+                    Simd p_00 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_00);
+                    Simd p_01 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_01);
+                    Simd p_02 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_02);
+                    Simd p_10 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_10);
+                    Simd p_11 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_11);
+                    Simd p_12 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_12);
+                    Simd p_20 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_00);
+                    Simd p_21 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_01);
+                    Simd p_22 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_02);
+                    Simd p_30 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_10);
+                    Simd p_31 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_11);
+                    Simd p_32 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_12);
+
+                    vstream(chi[ss+v]()(0)(0), p_00);
+                    vstream(chi[ss+v]()(0)(1), p_01);
+                    vstream(chi[ss+v]()(0)(2), p_02);
+                    vstream(chi[ss+v]()(1)(0), p_10);
+                    vstream(chi[ss+v]()(1)(1), p_11);
+                    vstream(chi[ss+v]()(1)(2), p_12);
+                    vstream(chi[ss+v]()(2)(0), p_20);
+                    vstream(chi[ss+v]()(2)(1), p_21);
+                    vstream(chi[ss+v]()(2)(2), p_22);
+                    vstream(chi[ss+v]()(3)(0), p_30);
+                    vstream(chi[ss+v]()(3)(1), p_31);
+                    vstream(chi[ss+v]()(3)(2), p_32);
+                }
+
+            #endif
+        }
+
+        this->M5Dtime += usecond();
+    }
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::M5Ddag(const FermionField& psi, const FermionField& phi,
+        FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+    {
+        GridBase* grid = psi._grid;
+        int Ls  = this->Ls;
+        int LLs = grid->_rdimensions[0];
+        int nsimd = Simd::Nsimd();
+
+        Vector<iSinglet<Simd> > u(LLs);
+        Vector<iSinglet<Simd> > l(LLs);
+        Vector<iSinglet<Simd> > d(LLs);
+
+        assert(Ls/LLs == nsimd);
+        assert(phi.checkerboard == psi.checkerboard);
+
+        chi.checkerboard = psi.checkerboard;
+
+        // just directly address via type pun
+        typedef typename Simd::scalar_type scalar_type;
+        scalar_type* u_p = (scalar_type*) &u[0];
+        scalar_type* l_p = (scalar_type*) &l[0];
+        scalar_type* d_p = (scalar_type*) &d[0];
+
+        for(int o=0; o<LLs; o++){ // outer
+        for(int i=0; i<nsimd; i++){ //inner
+            int s  = o + i*LLs;
+            int ss = o*nsimd + i;
+            u_p[ss] = upper[s];
+            l_p[ss] = lower[s];
+            d_p[ss] = diag[s];
+        }}
+
+        this->M5Dcalls++;
+        this->M5Dtime -= usecond();
+
+        parallel_for(int ss=0; ss<grid->oSites(); ss+=LLs){ // adds LLs
+
+        #if 0
+
+            alignas(64) SiteHalfSpinor hp;
+            alignas(64) SiteHalfSpinor hm;
+            alignas(64) SiteSpinor fp;
+            alignas(64) SiteSpinor fm;
+
+            for(int v=0; v<LLs; v++){
+
+                int vp = (v+1)%LLs;
+                int vm = (v+LLs-1)%LLs;
+
+                spProj5p(hp, psi[ss+vp]);
+                spProj5m(hm, psi[ss+vm]);
+
+                if(vp <= v){ rotate(hp, hp, 1); }
+                if(vm >= v){ rotate(hm, hm, nsimd-1); }
+
+                hp = hp*0.5;
+                hm = hm*0.5;
+                spRecon5p(fp, hp);
+                spRecon5m(fm, hm);
+
+                chi[ss+v] = d[v]*phi[ss+v]+u[v]*fp;
+                chi[ss+v] = chi[ss+v]     +l[v]*fm;
+            }
+
+        #else
+
+            for(int v=0; v<LLs; v++){
+
+                vprefetch(psi[ss+v+LLs]);
+
+                int vp = (v == LLs-1) ? 0     : v+1;
+                int vm = (v == 0    ) ? LLs-1 : v-1;
+
+                Simd hp_00 = psi[ss+vp]()(0)(0);
+                Simd hp_01 = psi[ss+vp]()(0)(1);
+                Simd hp_02 = psi[ss+vp]()(0)(2);
+                Simd hp_10 = psi[ss+vp]()(1)(0);
+                Simd hp_11 = psi[ss+vp]()(1)(1);
+                Simd hp_12 = psi[ss+vp]()(1)(2);
+
+                Simd hm_00 = psi[ss+vm]()(2)(0);
+                Simd hm_01 = psi[ss+vm]()(2)(1);
+                Simd hm_02 = psi[ss+vm]()(2)(2);
+                Simd hm_10 = psi[ss+vm]()(3)(0);
+                Simd hm_11 = psi[ss+vm]()(3)(1);
+                Simd hm_12 = psi[ss+vm]()(3)(2);
+
+                if (vp <= v){
+                    hp_00.v = Optimization::Rotate::tRotate<2>(hp_00.v);
+                    hp_01.v = Optimization::Rotate::tRotate<2>(hp_01.v);
+                    hp_02.v = Optimization::Rotate::tRotate<2>(hp_02.v);
+                    hp_10.v = Optimization::Rotate::tRotate<2>(hp_10.v);
+                    hp_11.v = Optimization::Rotate::tRotate<2>(hp_11.v);
+                    hp_12.v = Optimization::Rotate::tRotate<2>(hp_12.v);
+                }
+
+                if(vm >= v){
+                    hm_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_00.v);
+                    hm_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_01.v);
+                    hm_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_02.v);
+                    hm_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_10.v);
+                    hm_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_11.v);
+                    hm_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_12.v);
+                }
+
+                Simd p_00 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_00);
+                Simd p_01 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_01);
+                Simd p_02 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_02);
+                Simd p_10 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_10);
+                Simd p_11 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_11);
+                Simd p_12 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_12);
+                Simd p_20 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_00);
+                Simd p_21 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_01);
+                Simd p_22 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_02);
+                Simd p_30 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_10);
+                Simd p_31 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_11);
+                Simd p_32 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_12);
+
+                vstream(chi[ss+v]()(0)(0), p_00);
+                vstream(chi[ss+v]()(0)(1), p_01);
+                vstream(chi[ss+v]()(0)(2), p_02);
+                vstream(chi[ss+v]()(1)(0), p_10);
+                vstream(chi[ss+v]()(1)(1), p_11);
+                vstream(chi[ss+v]()(1)(2), p_12);
+                vstream(chi[ss+v]()(2)(0), p_20);
+                vstream(chi[ss+v]()(2)(1), p_21);
+                vstream(chi[ss+v]()(2)(2), p_22);
+                vstream(chi[ss+v]()(3)(0), p_30);
+                vstream(chi[ss+v]()(3)(1), p_31);
+                vstream(chi[ss+v]()(3)(2), p_32);
+            }
+        #endif
+
+        }
+
+        this->M5Dtime += usecond();
+    }
+
+    #ifdef AVX512
+        #include<simd/Intel512common.h>
+        #include<simd/Intel512avx.h>
+        #include<simd/Intel512single.h>
+    #endif
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInternalAsm(const FermionField& psi, FermionField& chi,
+        int LLs, int site, Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm)
+    {
+        #ifndef AVX512
+        {
+            SiteHalfSpinor BcastP;
+            SiteHalfSpinor BcastM;
+            SiteHalfSpinor SiteChiP;
+            SiteHalfSpinor SiteChiM;
+
+            // Ls*Ls * 2 * 12 * vol flops
+            for(int s1=0; s1<LLs; s1++){
+
+                for(int s2=0; s2<LLs; s2++){
+                for(int l=0; l < Simd::Nsimd(); l++){ // simd lane
+
+                    int s = s2 + l*LLs;
+                    int lex = s2 + LLs*site;
+
+                    if( s2==0 && l==0 ){
+                        SiteChiP=zero;
+                        SiteChiM=zero;
+                    }
+
+                    for(int sp=0; sp<2;  sp++){
+                    for(int co=0; co<Nc; co++){
+                        vbroadcast(BcastP()(sp)(co), psi[lex]()(sp)(co), l);
+                    }}
+
+                    for(int sp=0; sp<2;  sp++){
+                    for(int co=0; co<Nc; co++){
+                        vbroadcast(BcastM()(sp)(co), psi[lex]()(sp+2)(co), l);
+                    }}
+
+                    for(int sp=0; sp<2;  sp++){
+                    for(int co=0; co<Nc; co++){
+                        SiteChiP()(sp)(co) = real_madd(Matp[LLs*s+s1]()()(), BcastP()(sp)(co), SiteChiP()(sp)(co)); // 1100 us.
+                        SiteChiM()(sp)(co) = real_madd(Matm[LLs*s+s1]()()(), BcastM()(sp)(co), SiteChiM()(sp)(co)); // each found by commenting out
+                    }}
+                }}
+
+                {
+                    int lex = s1 + LLs*site;
+                    for(int sp=0; sp<2;  sp++){
+                    for(int co=0; co<Nc; co++){
+                        vstream(chi[lex]()(sp)(co),   SiteChiP()(sp)(co));
+                        vstream(chi[lex]()(sp+2)(co), SiteChiM()(sp)(co));
+                    }}
+                }
+            }
+
+        }
+        #else
+        {
+            // pointers
+            //  MASK_REGS;
+            #define Chi_00 %%zmm1
+            #define Chi_01 %%zmm2
+            #define Chi_02 %%zmm3
+            #define Chi_10 %%zmm4
+            #define Chi_11 %%zmm5
+            #define Chi_12 %%zmm6
+            #define Chi_20 %%zmm7
+            #define Chi_21 %%zmm8
+            #define Chi_22 %%zmm9
+            #define Chi_30 %%zmm10
+            #define Chi_31 %%zmm11
+            #define Chi_32 %%zmm12
+
+            #define BCAST0  %%zmm13
+            #define BCAST1  %%zmm14
+            #define BCAST2  %%zmm15
+            #define BCAST3  %%zmm16
+            #define BCAST4  %%zmm17
+            #define BCAST5  %%zmm18
+            #define BCAST6  %%zmm19
+            #define BCAST7  %%zmm20
+            #define BCAST8  %%zmm21
+            #define BCAST9  %%zmm22
+            #define BCAST10 %%zmm23
+            #define BCAST11 %%zmm24
+
+            int incr = LLs*LLs*sizeof(iSinglet<Simd>);
+            for(int s1=0; s1<LLs; s1++){
+
+                for(int s2=0; s2<LLs; s2++){
+
+                    int lex = s2 + LLs*site;
+                    uint64_t a0 = (uint64_t) &Matp[LLs*s2+s1]; // should be cacheable
+                    uint64_t a1 = (uint64_t) &Matm[LLs*s2+s1];
+                    uint64_t a2 = (uint64_t) &psi[lex];
+
+                    for(int l=0; l<Simd::Nsimd(); l++){ // simd lane
+                        if((s2+l)==0) {
+                            asm(
+                                    VPREFETCH1(0,%2)              VPREFETCH1(0,%1)
+                                    VPREFETCH1(12,%2)  	          VPREFETCH1(13,%2)
+                                    VPREFETCH1(14,%2)  	          VPREFETCH1(15,%2)
+                                    VBCASTCDUP(0,%2,BCAST0)
+                                    VBCASTCDUP(1,%2,BCAST1)
+                                    VBCASTCDUP(2,%2,BCAST2)
+                                    VBCASTCDUP(3,%2,BCAST3)
+                                    VBCASTCDUP(4,%2,BCAST4)       VMULMEM(0,%0,BCAST0,Chi_00)
+                                    VBCASTCDUP(5,%2,BCAST5)       VMULMEM(0,%0,BCAST1,Chi_01)
+                                    VBCASTCDUP(6,%2,BCAST6)       VMULMEM(0,%0,BCAST2,Chi_02)
+                                    VBCASTCDUP(7,%2,BCAST7)       VMULMEM(0,%0,BCAST3,Chi_10)
+                                    VBCASTCDUP(8,%2,BCAST8)       VMULMEM(0,%0,BCAST4,Chi_11)
+                                    VBCASTCDUP(9,%2,BCAST9)       VMULMEM(0,%0,BCAST5,Chi_12)
+                                    VBCASTCDUP(10,%2,BCAST10)     VMULMEM(0,%1,BCAST6,Chi_20)
+                                    VBCASTCDUP(11,%2,BCAST11)     VMULMEM(0,%1,BCAST7,Chi_21)
+                                    VMULMEM(0,%1,BCAST8,Chi_22)
+                                    VMULMEM(0,%1,BCAST9,Chi_30)
+                                    VMULMEM(0,%1,BCAST10,Chi_31)
+                                    VMULMEM(0,%1,BCAST11,Chi_32)
+                                    : : "r" (a0), "r" (a1), "r" (a2)                            );
+                        } else {
+                            asm(
+                                    VBCASTCDUP(0,%2,BCAST0)   VMADDMEM(0,%0,BCAST0,Chi_00)
+                                    VBCASTCDUP(1,%2,BCAST1)   VMADDMEM(0,%0,BCAST1,Chi_01)
+                                    VBCASTCDUP(2,%2,BCAST2)   VMADDMEM(0,%0,BCAST2,Chi_02)
+                                    VBCASTCDUP(3,%2,BCAST3)   VMADDMEM(0,%0,BCAST3,Chi_10)
+                                    VBCASTCDUP(4,%2,BCAST4)   VMADDMEM(0,%0,BCAST4,Chi_11)
+                                    VBCASTCDUP(5,%2,BCAST5)   VMADDMEM(0,%0,BCAST5,Chi_12)
+                                    VBCASTCDUP(6,%2,BCAST6)   VMADDMEM(0,%1,BCAST6,Chi_20)
+                                    VBCASTCDUP(7,%2,BCAST7)   VMADDMEM(0,%1,BCAST7,Chi_21)
+                                    VBCASTCDUP(8,%2,BCAST8)   VMADDMEM(0,%1,BCAST8,Chi_22)
+                                    VBCASTCDUP(9,%2,BCAST9)   VMADDMEM(0,%1,BCAST9,Chi_30)
+                                    VBCASTCDUP(10,%2,BCAST10) VMADDMEM(0,%1,BCAST10,Chi_31)
+                                    VBCASTCDUP(11,%2,BCAST11) VMADDMEM(0,%1,BCAST11,Chi_32)
+                                    : : "r" (a0), "r" (a1), "r" (a2)                            );
+                        }
+                        a0 = a0 + incr;
+                        a1 = a1 + incr;
+                        a2 = a2 + sizeof(Simd::scalar_type);
+                    }
+                }
+
+                {
+                  int lexa = s1+LLs*site;
+                  asm (
+                     VSTORE(0,%0,Chi_00) VSTORE(1 ,%0,Chi_01)  VSTORE(2 ,%0,Chi_02)
+                     VSTORE(3,%0,Chi_10) VSTORE(4 ,%0,Chi_11)  VSTORE(5 ,%0,Chi_12)
+                     VSTORE(6,%0,Chi_20) VSTORE(7 ,%0,Chi_21)  VSTORE(8 ,%0,Chi_22)
+                     VSTORE(9,%0,Chi_30) VSTORE(10,%0,Chi_31)  VSTORE(11,%0,Chi_32)
+                     : : "r" ((uint64_t)&chi[lexa]) : "memory" );
+
+                }
+            }
+        }
+
+        #undef Chi_00
+        #undef Chi_01
+        #undef Chi_02
+        #undef Chi_10
+        #undef Chi_11
+        #undef Chi_12
+        #undef Chi_20
+        #undef Chi_21
+        #undef Chi_22
+        #undef Chi_30
+        #undef Chi_31
+        #undef Chi_32
+
+        #undef BCAST0
+        #undef BCAST1
+        #undef BCAST2
+        #undef BCAST3
+        #undef BCAST4
+        #undef BCAST5
+        #undef BCAST6
+        #undef BCAST7
+        #undef BCAST8
+        #undef BCAST9
+        #undef BCAST10
+        #undef BCAST11
+        #endif
+    };
+
+    // Z-mobius version
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInternalZAsm(const FermionField& psi, FermionField& chi,
+        int LLs, int site, Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm)
+    {
+        std::cout << "Error: zMobius not implemented for EOFA" << std::endl;
+        exit(-1);
+    };
+
+    template<class Impl>
+    void DomainWallEOFAFermion<Impl>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv)
+    {
+        int Ls  = this->Ls;
+        int LLs = psi._grid->_rdimensions[0];
+        int vol = psi._grid->oSites()/LLs;
+
+        chi.checkerboard = psi.checkerboard;
+
+        Vector<iSinglet<Simd> > Matp;
+        Vector<iSinglet<Simd> > Matm;
+        Vector<iSinglet<Simd> > *_Matp;
+        Vector<iSinglet<Simd> > *_Matm;
+
+        //  MooeeInternalCompute(dag,inv,Matp,Matm);
+        if(inv && dag){
+            _Matp = &this->MatpInvDag;
+            _Matm = &this->MatmInvDag;
+        }
+
+        if(inv && (!dag)){
+            _Matp = &this->MatpInv;
+            _Matm = &this->MatmInv;
+        }
+
+        if(!inv){
+            MooeeInternalCompute(dag, inv, Matp, Matm);
+            _Matp = &Matp;
+            _Matm = &Matm;
+        }
+
+        assert(_Matp->size() == Ls*LLs);
+
+        this->MooeeInvCalls++;
+        this->MooeeInvTime -= usecond();
+
+        if(switcheroo<Coeff_t>::iscomplex()){
+            parallel_for(auto site=0; site<vol; site++){
+                MooeeInternalZAsm(psi, chi, LLs, site, *_Matp, *_Matm);
+            }
+        } else {
+            parallel_for(auto site=0; site<vol; site++){
+                MooeeInternalAsm(psi, chi, LLs, site, *_Matp, *_Matm);
+            }
+        }
+
+        this->MooeeInvTime += usecond();
+    }
+
+    #ifdef DOMAIN_WALL_EOFA_DPERP_VEC
+
+        INSTANTIATE_DPERP_DWF_EOFA(DomainWallVec5dImplD);
+        INSTANTIATE_DPERP_DWF_EOFA(DomainWallVec5dImplF);
+        INSTANTIATE_DPERP_DWF_EOFA(ZDomainWallVec5dImplD);
+        INSTANTIATE_DPERP_DWF_EOFA(ZDomainWallVec5dImplF);
+
+        INSTANTIATE_DPERP_DWF_EOFA(DomainWallVec5dImplDF);
+        INSTANTIATE_DPERP_DWF_EOFA(DomainWallVec5dImplFH);
+        INSTANTIATE_DPERP_DWF_EOFA(ZDomainWallVec5dImplDF);
+        INSTANTIATE_DPERP_DWF_EOFA(ZDomainWallVec5dImplFH);
+
+        template void DomainWallEOFAFermion<DomainWallVec5dImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<DomainWallVec5dImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<ZDomainWallVec5dImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<ZDomainWallVec5dImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+
+        template void DomainWallEOFAFermion<DomainWallVec5dImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<DomainWallVec5dImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<ZDomainWallVec5dImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+        template void DomainWallEOFAFermion<ZDomainWallVec5dImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+
+    #endif
+
+}}
diff --git a/lib/qcd/action/fermion/Fermion.h b/lib/qcd/action/fermion/Fermion.h
index 0f803f44..ac2a94b2 100644
--- a/lib/qcd/action/fermion/Fermion.h
+++ b/lib/qcd/action/fermion/Fermion.h
@@ -55,7 +55,7 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
 #include <Grid/qcd/action/fermion/ImprovedStaggeredFermion5D.h>
 #include <Grid/qcd/action/fermion/CayleyFermion5D.h>     // Cayley types
 #include <Grid/qcd/action/fermion/DomainWallFermion.h>
-#include <Grid/qcd/action/fermion/DomainWallFermion.h>
+#include <Grid/qcd/action/fermion/DomainWallEOFAFermion.h>
 #include <Grid/qcd/action/fermion/MobiusFermion.h>
 #include <Grid/qcd/action/fermion/ZMobiusFermion.h>
 #include <Grid/qcd/action/fermion/SchurDiagTwoKappa.h>
@@ -113,6 +113,14 @@ typedef DomainWallFermion<WilsonImplRL> DomainWallFermionRL;
 typedef DomainWallFermion<WilsonImplFH> DomainWallFermionFH;
 typedef DomainWallFermion<WilsonImplDF> DomainWallFermionDF;
 
+typedef DomainWallEOFAFermion<WilsonImplR> DomainWallEOFAFermionR;
+typedef DomainWallEOFAFermion<WilsonImplF> DomainWallEOFAFermionF;
+typedef DomainWallEOFAFermion<WilsonImplD> DomainWallEOFAFermionD;
+
+typedef DomainWallEOFAFermion<WilsonImplRL> DomainWallEOFAFermionRL;
+typedef DomainWallEOFAFermion<WilsonImplFH> DomainWallEOFAFermionFH;
+typedef DomainWallEOFAFermion<WilsonImplDF> DomainWallEOFAFermionDF;
+
 typedef MobiusFermion<WilsonImplR> MobiusFermionR;
 typedef MobiusFermion<WilsonImplF> MobiusFermionF;
 typedef MobiusFermion<WilsonImplD> MobiusFermionD;
@@ -138,6 +146,14 @@ typedef DomainWallFermion<DomainWallVec5dImplRL> DomainWallFermionVec5dRL;
 typedef DomainWallFermion<DomainWallVec5dImplFH> DomainWallFermionVec5dFH;
 typedef DomainWallFermion<DomainWallVec5dImplDF> DomainWallFermionVec5dDF;
 
+typedef DomainWallEOFAFermion<DomainWallVec5dImplR> DomainWallEOFAFermionVec5dR;
+typedef DomainWallEOFAFermion<DomainWallVec5dImplF> DomainWallEOFAFermionVec5dF;
+typedef DomainWallEOFAFermion<DomainWallVec5dImplD> DomainWallEOFAFermionVec5dD;
+
+typedef DomainWallEOFAFermion<DomainWallVec5dImplRL> DomainWallEOFAFermionVec5dRL;
+typedef DomainWallEOFAFermion<DomainWallVec5dImplFH> DomainWallEOFAFermionVec5dFH;
+typedef DomainWallEOFAFermion<DomainWallVec5dImplDF> DomainWallEOFAFermionVec5dDF;
+
 typedef MobiusFermion<DomainWallVec5dImplR> MobiusFermionVec5dR;
 typedef MobiusFermion<DomainWallVec5dImplF> MobiusFermionVec5dF;
 typedef MobiusFermion<DomainWallVec5dImplD> MobiusFermionVec5dD;
@@ -206,6 +222,14 @@ typedef DomainWallFermion<GparityWilsonImplRL> GparityDomainWallFermionRL;
 typedef DomainWallFermion<GparityWilsonImplFH> GparityDomainWallFermionFH;
 typedef DomainWallFermion<GparityWilsonImplDF> GparityDomainWallFermionDF;
 
+typedef DomainWallEOFAFermion<GparityWilsonImplR> GparityDomainWallEOFAFermionR;
+typedef DomainWallEOFAFermion<GparityWilsonImplF> GparityDomainWallEOFAFermionF;
+typedef DomainWallEOFAFermion<GparityWilsonImplD> GparityDomainWallEOFAFermionD;
+
+typedef DomainWallEOFAFermion<GparityWilsonImplRL> GparityDomainWallEOFAFermionRL;
+typedef DomainWallEOFAFermion<GparityWilsonImplFH> GparityDomainWallEOFAFermionFH;
+typedef DomainWallEOFAFermion<GparityWilsonImplDF> GparityDomainWallEOFAFermionDF;
+
 typedef WilsonTMFermion<GparityWilsonImplR> GparityWilsonTMFermionR;
 typedef WilsonTMFermion<GparityWilsonImplF> GparityWilsonTMFermionF;
 typedef WilsonTMFermion<GparityWilsonImplD> GparityWilsonTMFermionD;
diff --git a/lib/qcd/modules/FermionOperatorModules.h b/lib/qcd/modules/FermionOperatorModules.h
index c66842c6..fc9d96a7 100644
--- a/lib/qcd/modules/FermionOperatorModules.h
+++ b/lib/qcd/modules/FermionOperatorModules.h
@@ -72,7 +72,7 @@ protected:
   }
 
   virtual unsigned int Ls(){
-    return 0;  
+    return 0;
   }
 
   virtual void print_parameters(){
@@ -97,7 +97,7 @@ class HMC_FermionOperatorModuleFactory
     : public Factory < FermionOperatorModuleBase<QCD::FermionOperator<FermionImpl> > ,  Reader<ReaderClass> > {
  public:
   // use SINGLETON FUNCTOR MACRO HERE
-  typedef Reader<ReaderClass> TheReader; 
+  typedef Reader<ReaderClass> TheReader;
 
   HMC_FermionOperatorModuleFactory(const HMC_FermionOperatorModuleFactory& e) = delete;
   void operator=(const HMC_FermionOperatorModuleFactory& e) = delete;
@@ -122,7 +122,7 @@ namespace QCD{
 // Modules
 class WilsonFermionParameters : Serializable {
  public:
-  GRID_SERIALIZABLE_CLASS_MEMBERS(WilsonFermionParameters, 
+  GRID_SERIALIZABLE_CLASS_MEMBERS(WilsonFermionParameters,
     RealD, mass);
 };
 
@@ -144,7 +144,7 @@ class WilsonFermionModule: public FermionOperatorModule<WilsonFermion, FermionIm
 
 class MobiusFermionParameters : Serializable {
  public:
-  GRID_SERIALIZABLE_CLASS_MEMBERS(MobiusFermionParameters, 
+  GRID_SERIALIZABLE_CLASS_MEMBERS(MobiusFermionParameters,
     RealD, mass,
     RealD, M5,
     RealD, b,
@@ -166,7 +166,7 @@ class MobiusFermionModule: public FermionOperatorModule<MobiusFermion, FermionIm
     auto GridMod = this->GridRefs[0];
     auto GridMod5d = this->GridRefs[1];
     typename FermionImpl::GaugeField U(GridMod->get_full());
-    this->FOPtr.reset(new MobiusFermion<FermionImpl>( U, *(GridMod->get_full()), *(GridMod->get_rb()), 
+    this->FOPtr.reset(new MobiusFermion<FermionImpl>( U, *(GridMod->get_full()), *(GridMod->get_rb()),
                                                       *(GridMod5d->get_full()), *(GridMod5d->get_rb()),
                                                       this->Par_.mass, this->Par_.M5, this->Par_.b, this->Par_.c));
   }
@@ -175,7 +175,7 @@ class MobiusFermionModule: public FermionOperatorModule<MobiusFermion, FermionIm
 
 class DomainWallFermionParameters : Serializable {
  public:
-  GRID_SERIALIZABLE_CLASS_MEMBERS(DomainWallFermionParameters, 
+  GRID_SERIALIZABLE_CLASS_MEMBERS(DomainWallFermionParameters,
     RealD, mass,
     RealD, M5,
     unsigned int, Ls);
@@ -195,16 +195,49 @@ class DomainWallFermionModule: public FermionOperatorModule<DomainWallFermion, F
     auto GridMod = this->GridRefs[0];
     auto GridMod5d = this->GridRefs[1];
     typename FermionImpl::GaugeField U(GridMod->get_full());
-    this->FOPtr.reset(new DomainWallFermion<FermionImpl>( U, *(GridMod->get_full()), *(GridMod->get_rb()), 
+    this->FOPtr.reset(new DomainWallFermion<FermionImpl>( U, *(GridMod->get_full()), *(GridMod->get_rb()),
                                                       *(GridMod5d->get_full()), *(GridMod5d->get_rb()),
                                                       this->Par_.mass, this->Par_.M5));
   }
 };
 
 
+class DomainWallEOFAFermionParameters : Serializable {
+ public:
+  GRID_SERIALIZABLE_CLASS_MEMBERS(DomainWallEOFAFermionParameters,
+    RealD, mq1,
+    RealD, mq2,
+    RealD, mq3,
+    RealD, shift,
+    int, pm,
+    RealD, M5,
+    unsigned int, Ls);
+};
+
+template <class FermionImpl >
+class DomainWallEOFAFermionModule: public FermionOperatorModule<DomainWallEOFAFermion, FermionImpl, DomainWallEOFAFermionParameters> {
+  typedef FermionOperatorModule<DomainWallEOFAFermion, FermionImpl, DomainWallEOFAFermionParameters> FermBase;
+  using FermBase::FermBase; // for constructors
+
+  virtual unsigned int Ls(){
+    return this->Par_.Ls;
+  }
+
+  // acquire resource
+  virtual void initialize(){
+    auto GridMod = this->GridRefs[0];
+    auto GridMod5d = this->GridRefs[1];
+    typename FermionImpl::GaugeField U(GridMod->get_full());
+    this->FOPtr.reset(new DomainWallEOFAFermion<FermionImpl>( U, *(GridMod->get_full()), *(GridMod->get_rb()),
+                                                      *(GridMod5d->get_full()), *(GridMod5d->get_rb()),
+                                                      this->Par_.mq1, this->Par_.mq2, this->Par_.mq3,
+                                                      this->Par_.shift, this->Par_.pm, this->Par_.M5));
+  }
+};
+
 
 } // QCD
 } // Grid
 
 
-#endif //FERMIONOPERATOR_MODULES_H
\ No newline at end of file
+#endif //FERMIONOPERATOR_MODULES_H
diff --git a/tests/core/Test_dwf_eofa_even_odd.cc b/tests/core/Test_dwf_eofa_even_odd.cc
new file mode 100644
index 00000000..5fe0f653
--- /dev/null
+++ b/tests/core/Test_dwf_eofa_even_odd.cc
@@ -0,0 +1,239 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/core/Test_dwf_eofa_even_odd.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+template<class d>
+struct scal {
+    d internal;
+};
+
+Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT
+};
+
+int main (int argc, char ** argv)
+{
+    Grid_init(&argc, &argv);
+
+    int threads = GridThread::GetThreads();
+    std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+    const int Ls = 8;
+    // GridCartesian*         UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+    GridCartesian*         UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+    GridCartesian*         FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+    GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+    GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+    std::vector<int> seeds4({1,2,3,4});
+    std::vector<int> seeds5({5,6,7,8});
+
+    GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4);
+    GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds5);
+
+    LatticeFermion    src   (FGrid); random(RNG5, src);
+    LatticeFermion    phi   (FGrid); random(RNG5, phi);
+    LatticeFermion    chi   (FGrid); random(RNG5, chi);
+    LatticeFermion    result(FGrid); result = zero;
+    LatticeFermion    ref   (FGrid); ref = zero;
+    LatticeFermion    tmp   (FGrid); tmp = zero;
+    LatticeFermion    err   (FGrid); err = zero;
+    LatticeGaugeField Umu   (UGrid); SU3::HotConfiguration(RNG4, Umu);
+    std::vector<LatticeColourMatrix> U(4,UGrid);
+
+    // Only one non-zero (y)
+    Umu = zero;
+    for(int nn=0; nn<Nd; nn++){
+        random(RNG4, U[nn]);
+        if(nn>0){ U[nn] = zero; }
+        PokeIndex<LorentzIndex>(Umu, U[nn], nn);
+    }
+
+    RealD mq1   = 0.1;
+    RealD mq2   = 0.5;
+    RealD mq3   = 1.0;
+    RealD shift = 0.1234;
+    RealD M5    = 1.8;
+    int   pm    = 1;
+    DomainWallEOFAFermionR Ddwf(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mq1, mq2, mq3, shift, pm, M5);
+
+    LatticeFermion src_e (FrbGrid);
+    LatticeFermion src_o (FrbGrid);
+    LatticeFermion r_e   (FrbGrid);
+    LatticeFermion r_o   (FrbGrid);
+    LatticeFermion r_eo  (FGrid);
+    LatticeFermion r_eeoo(FGrid);
+
+    std::cout << GridLogMessage << "==========================================================" << std::endl;
+    std::cout << GridLogMessage << "= Testing that Meo + Moe + Moo + Mee = Munprec " << std::endl;
+    std::cout << GridLogMessage << "==========================================================" << std::endl;
+
+    pickCheckerboard(Even, src_e, src);
+    pickCheckerboard(Odd,  src_o, src);
+
+    Ddwf.Meooe(src_e, r_o); std::cout << GridLogMessage << "Applied Meo" << std::endl;
+    Ddwf.Meooe(src_o, r_e); std::cout << GridLogMessage << "Applied Moe" << std::endl;
+    setCheckerboard(r_eo, r_o);
+    setCheckerboard(r_eo, r_e);
+
+    Ddwf.Mooee(src_e, r_e); std::cout << GridLogMessage << "Applied Mee" << std::endl;
+    Ddwf.Mooee(src_o, r_o); std::cout << GridLogMessage << "Applied Moo" << std::endl;
+    setCheckerboard(r_eeoo, r_e);
+    setCheckerboard(r_eeoo, r_o);
+
+    r_eo = r_eo + r_eeoo;
+    Ddwf.M(src, ref);
+
+    // std::cout << GridLogMessage << r_eo << std::endl;
+    // std::cout << GridLogMessage << ref  << std::endl;
+
+    err = ref - r_eo;
+    std::cout << GridLogMessage << "EO norm diff   " << norm2(err) << " " << norm2(ref) << " " << norm2(r_eo) << std::endl;
+
+    LatticeComplex cerr(FGrid);
+    cerr = localInnerProduct(err,err);
+    // std::cout << GridLogMessage << cerr << std::endl;
+
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+    std::cout << GridLogMessage << "= Test Ddagger is the dagger of D by requiring                " << std::endl;
+    std::cout << GridLogMessage << "=  < phi | Deo | chi > * = < chi | Deo^dag| phi>  " << std::endl;
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+
+    LatticeFermion chi_e (FrbGrid);
+    LatticeFermion chi_o (FrbGrid);
+
+    LatticeFermion dchi_e(FrbGrid);
+    LatticeFermion dchi_o(FrbGrid);
+
+    LatticeFermion phi_e (FrbGrid);
+    LatticeFermion phi_o (FrbGrid);
+
+    LatticeFermion dphi_e(FrbGrid);
+    LatticeFermion dphi_o(FrbGrid);
+
+    pickCheckerboard(Even, chi_e, chi);
+    pickCheckerboard(Odd , chi_o, chi);
+    pickCheckerboard(Even, phi_e, phi);
+    pickCheckerboard(Odd , phi_o, phi);
+
+    Ddwf.Meooe   (chi_e, dchi_o);
+    Ddwf.Meooe   (chi_o, dchi_e);
+    Ddwf.MeooeDag(phi_e, dphi_o);
+    Ddwf.MeooeDag(phi_o, dphi_e);
+
+    ComplexD pDce = innerProduct(phi_e, dchi_e);
+    ComplexD pDco = innerProduct(phi_o, dchi_o);
+    ComplexD cDpe = innerProduct(chi_e, dphi_e);
+    ComplexD cDpo = innerProduct(chi_o, dphi_o);
+
+    std::cout << GridLogMessage << "e " << pDce << " " << cDpe << std::endl;
+    std::cout << GridLogMessage << "o " << pDco << " " << cDpo << std::endl;
+
+    std::cout << GridLogMessage << "pDce - conj(cDpo) " << pDce-conj(cDpo) << std::endl;
+    std::cout << GridLogMessage << "pDco - conj(cDpe) " << pDco-conj(cDpe) << std::endl;
+
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+    std::cout << GridLogMessage << "= Test MeeInv Mee = 1                                         " << std::endl;
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+
+    pickCheckerboard(Even, chi_e, chi);
+    pickCheckerboard(Odd , chi_o, chi);
+
+    Ddwf.Mooee   (chi_e, src_e);
+    Ddwf.MooeeInv(src_e, phi_e);
+
+    Ddwf.Mooee   (chi_o, src_o);
+    Ddwf.MooeeInv(src_o, phi_o);
+
+    setCheckerboard(phi, phi_e);
+    setCheckerboard(phi, phi_o);
+
+    err = phi - chi;
+    std::cout << GridLogMessage << "norm diff   " << norm2(err) << std::endl;
+
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+    std::cout << GridLogMessage << "= Test MeeInvDag MeeDag = 1                                   " << std::endl;
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+
+    pickCheckerboard(Even, chi_e, chi);
+    pickCheckerboard(Odd , chi_o, chi);
+
+    Ddwf.MooeeDag   (chi_e, src_e);
+    Ddwf.MooeeInvDag(src_e, phi_e);
+
+    Ddwf.MooeeDag   (chi_o, src_o);
+    Ddwf.MooeeInvDag(src_o, phi_o);
+
+    setCheckerboard(phi, phi_e);
+    setCheckerboard(phi, phi_o);
+
+    err = phi - chi;
+    std::cout << GridLogMessage << "norm diff   " << norm2(err) << std::endl;
+
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+    std::cout << GridLogMessage << "= Test MpcDagMpc is Hermitian              " << std::endl;
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+
+    random(RNG5, phi);
+    random(RNG5, chi);
+    pickCheckerboard(Even, chi_e, chi);
+    pickCheckerboard(Odd , chi_o, chi);
+    pickCheckerboard(Even, phi_e, phi);
+    pickCheckerboard(Odd , phi_o, phi);
+    RealD t1,t2;
+
+    SchurDiagMooeeOperator<DomainWallEOFAFermionR,LatticeFermion> HermOpEO(Ddwf);
+    HermOpEO.MpcDagMpc(chi_e, dchi_e, t1, t2);
+    HermOpEO.MpcDagMpc(chi_o, dchi_o, t1, t2);
+
+    HermOpEO.MpcDagMpc(phi_e, dphi_e, t1, t2);
+    HermOpEO.MpcDagMpc(phi_o, dphi_o, t1, t2);
+
+    pDce = innerProduct(phi_e, dchi_e);
+    pDco = innerProduct(phi_o, dchi_o);
+    cDpe = innerProduct(chi_e, dphi_e);
+    cDpo = innerProduct(chi_o, dphi_o);
+
+    std::cout << GridLogMessage << "e " << pDce << " " << cDpe << std::endl;
+    std::cout << GridLogMessage << "o " << pDco << " " << cDpo << std::endl;
+
+    std::cout << GridLogMessage << "pDce - conj(cDpo) " << pDco-conj(cDpo) << std::endl;
+    std::cout << GridLogMessage << "pDco - conj(cDpe) " << pDce-conj(cDpe) << std::endl;
+
+    Grid_finalize();
+}
diff --git a/tests/debug/Test_reweight_dwf_eofa.cc b/tests/debug/Test_reweight_dwf_eofa.cc
new file mode 100644
index 00000000..98a17e2f
--- /dev/null
+++ b/tests/debug/Test_reweight_dwf_eofa.cc
@@ -0,0 +1,206 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/debug/Test_reweight_dwf_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+// parameters for test
+const std::vector<int> grid_dim = { 8, 8, 8, 8 };
+const int Ls = 8;
+const int Nhits = 25;
+const int max_iter = 5000;
+const RealD mf = 0.1;
+const RealD mb = 0.11;
+const RealD M5 = 1.8;
+const RealD stop_tol = 1.0e-12;
+
+RealD mean(const std::vector<RealD>& data)
+{
+    int N = data.size();
+    RealD mean(0.0);
+    for(int i=0; i<N; ++i){ mean += data[i]; }
+    return mean/RealD(N);
+}
+
+RealD jack_mean(const std::vector<RealD>& data, int sample)
+{
+    int N = data.size();
+    RealD mean(0.0);
+    for(int i=0; i<N; ++i){ if(i != sample){ mean += data[i]; } }
+    return mean/RealD(N-1);
+}
+
+RealD jack_std(const std::vector<RealD>& jacks, RealD mean)
+{
+    int N = jacks.size();
+    RealD std(0.0);
+    for(int i=0; i<N; ++i){ std += std::pow(jacks[i]-mean, 2.0); }
+    return std::sqrt(RealD(N-1)/RealD(N)*std);
+}
+
+std::vector<RealD> jack_stats(const std::vector<RealD>& data)
+{
+    int N = data.size();
+    std::vector<RealD> jack_samples(N);
+    std::vector<RealD> jack_stats(2);
+
+    jack_stats[0] = mean(data);
+    for(int i=0; i<N; i++){ jack_samples[i] = jack_mean(data,i); }
+    jack_stats[1] = jack_std(jack_samples, jack_stats[0]);
+    return jack_stats;
+}
+
+int main(int argc, char **argv)
+{
+  Grid_init(&argc, &argv);
+
+  // Initialize spacetime grid
+  std::cout << GridLogMessage << "Lattice dimensions: "
+    << grid_dim << "   Ls: " << Ls << std::endl;
+  GridCartesian* UGrid = SpaceTimeGrid::makeFourDimGrid(grid_dim,
+      GridDefaultSimd(Nd, vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian* FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Set up RNGs
+  std::vector<int> seeds4({1, 2, 3, 4});
+  std::vector<int> seeds5({5, 6, 7, 8});
+  GridParallelRNG RNG5(FGrid);
+  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);
+  RNG4.SeedFixedIntegers(seeds4);
+
+  // Random gauge field
+  LatticeGaugeField Umu(UGrid);
+  SU3::HotConfiguration(RNG4, Umu);
+
+  // Initialize RHMC fermion operators
+  DomainWallFermionR Ddwf_f(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, M5);
+  DomainWallFermionR Ddwf_b(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, M5);
+  SchurDiagMooeeOperator<DomainWallFermionR, LatticeFermion> MdagM(Ddwf_f);
+  SchurDiagMooeeOperator<DomainWallFermionR, LatticeFermion> VdagV(Ddwf_b);
+
+  // Degree 12 rational approximations to x^(1/4) and x^(-1/4)
+  double     lo = 0.0001;
+  double     hi = 95.0;
+  int precision = 64;
+  int    degree = 12;
+  AlgRemez remez(lo, hi, precision);
+  std::cout << GridLogMessage << "Generating degree " << degree << " for x^(1/4)" << std::endl;
+  remez.generateApprox(degree, 1, 4);
+  MultiShiftFunction PowerQuarter(remez, stop_tol, false);
+  MultiShiftFunction PowerNegQuarter(remez, stop_tol, true);
+
+  // Stochastically estimate reweighting factor via RHMC
+  RealD scale = std::sqrt(0.5);
+  std::vector<RealD> rw_rhmc(Nhits);
+  ConjugateGradientMultiShift<LatticeFermion> msCG_V(max_iter, PowerQuarter);
+  ConjugateGradientMultiShift<LatticeFermion> msCG_M(max_iter, PowerNegQuarter);
+  std::cout.precision(12);
+
+  for(int hit=0; hit<Nhits; hit++){
+
+    // Gaussian source
+    LatticeFermion Phi    (Ddwf_f.FermionGrid());
+    LatticeFermion PhiOdd (Ddwf_f.FermionRedBlackGrid());
+    std::vector<LatticeFermion> tmp(2, Ddwf_f.FermionRedBlackGrid());
+    gaussian(RNG5, Phi);
+    Phi = Phi*scale;
+
+    pickCheckerboard(Odd, PhiOdd, Phi);
+
+    // evaluate -log(rw)
+    msCG_V(VdagV, PhiOdd, tmp[0]);
+    msCG_M(MdagM, tmp[0], tmp[1]);
+    rw_rhmc[hit] = norm2(tmp[1]) - norm2(PhiOdd);
+    std::cout << std::endl << "==================================================" << std::endl;
+    std::cout << " --- RHMC: Hit " << hit << ": rw = " << rw_rhmc[hit];
+    std::cout << std::endl << "==================================================" << std::endl << std::endl;
+
+  }
+
+  // Initialize EOFA fermion operators
+  RealD shift_L = 0.0;
+  RealD shift_R = -1.0;
+  int pm = 1;
+  DomainWallEOFAFermionR Deofa_L(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, mf, mb, shift_L, pm, M5);
+  DomainWallEOFAFermionR Deofa_R(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, mf, mb, shift_R, pm, M5);
+  MdagMLinearOperator<DomainWallEOFAFermionR, LatticeFermion> LdagL(Deofa_L);
+  MdagMLinearOperator<DomainWallEOFAFermionR, LatticeFermion> RdagR(Deofa_R);
+
+  // Stochastically estimate reweighting factor via EOFA
+  RealD k = Deofa_L.k;
+  std::vector<RealD> rw_eofa(Nhits);
+  ConjugateGradient<LatticeFermion> CG(stop_tol, max_iter);
+  SchurRedBlackDiagMooeeSolve<LatticeFermion> SchurSolver(CG);
+
+  for(int hit=0; hit<Nhits; hit++){
+
+    // Gaussian source
+    LatticeFermion Phi       (Deofa_L.FermionGrid());
+    LatticeFermion spProj_Phi(Deofa_L.FermionGrid());
+    std::vector<LatticeFermion> tmp(2, Deofa_L.FermionGrid());
+    gaussian(RNG5, Phi);
+    Phi = Phi*scale;
+
+    // evaluate -log(rw)
+    // LH term
+    for(int s=0; s<Ls; ++s){ axpby_ssp_pminus(spProj_Phi, 0.0, Phi, 1.0, Phi, s, s); }
+    Deofa_L.Omega(spProj_Phi, tmp[0], -1, 0);
+    G5R5(tmp[1], tmp[0]);
+    tmp[0] = zero;
+    SchurSolver(Deofa_L, tmp[1], tmp[0]);
+    Deofa_L.Omega(tmp[0], tmp[1], -1, 1);
+    rw_eofa[hit] = -k*innerProduct(spProj_Phi,tmp[1]).real();
+
+    // RH term
+    for(int s=0; s<Ls; ++s){ axpby_ssp_pplus(spProj_Phi, 0.0, Phi, 1.0, Phi, s, s); }
+    Deofa_R.Omega(spProj_Phi, tmp[0], 1, 0);
+    G5R5(tmp[1], tmp[0]);
+    tmp[0] = zero;
+    SchurSolver(Deofa_R, tmp[1], tmp[0]);
+    Deofa_R.Omega(tmp[0], tmp[1], 1, 1);
+    rw_eofa[hit] += k*innerProduct(spProj_Phi,tmp[1]).real();
+    std::cout << std::endl << "==================================================" << std::endl;
+    std::cout << " --- EOFA: Hit " << hit << ": rw = " << rw_eofa[hit];
+    std::cout << std::endl << "==================================================" << std::endl << std::endl;
+
+  }
+
+  std::vector<RealD> rhmc_result = jack_stats(rw_rhmc);
+  std::vector<RealD> eofa_result = jack_stats(rw_eofa);
+  std::cout << std::endl << "RHMC: rw = " << rhmc_result[0] << " +/- " << rhmc_result[1] << std::endl;
+  std::cout << std::endl << "EOFA: rw = " << eofa_result[0] << " +/- " << eofa_result[1] << std::endl;
+
+  Grid_finalize();
+}

From b7f93aeb4de96ecadcc58bbf6afcb61decfd6a35 Mon Sep 17 00:00:00 2001
From: David Murphy <david.murphy@ligo.org>
Date: Tue, 15 Aug 2017 14:18:51 -0400
Subject: [PATCH 033/115] Change CayleyFermion5D::SetCoefficientsInternal to
 virtual to allow overriding in derived EOFA classes

---
 lib/qcd/action/fermion/CayleyFermion5D.h | 56 ++++++++++++------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/lib/qcd/action/fermion/CayleyFermion5D.h b/lib/qcd/action/fermion/CayleyFermion5D.h
index cce13e12..ef75235a 100644
--- a/lib/qcd/action/fermion/CayleyFermion5D.h
+++ b/lib/qcd/action/fermion/CayleyFermion5D.h
@@ -1,6 +1,6 @@
     /*************************************************************************************
 
-    Grid physics library, www.github.com/paboyle/Grid 
+    Grid physics library, www.github.com/paboyle/Grid
 
     Source file: ./lib/qcd/action/fermion/CayleyFermion5D.h
 
@@ -35,24 +35,24 @@ namespace Grid {
 
   namespace QCD {
 
-     template<typename T> struct switcheroo   {  
-       static inline int iscomplex()  { return 0; } 
+     template<typename T> struct switcheroo   {
+       static inline int iscomplex()  { return 0; }
 
        template<class vec>
        static inline vec mult(vec a, vec b) {
 	 return real_mult(a,b);
        }
      };
-     template<> struct switcheroo<ComplexD> {  
-       static inline int iscomplex()  { return 1; } 
+     template<> struct switcheroo<ComplexD> {
+       static inline int iscomplex()  { return 1; }
 
        template<class vec>
        static inline vec mult(vec a, vec b) {
 	 return a*b;
        }
      };
-     template<> struct switcheroo<ComplexF> {  
-       static inline int iscomplex()  { return 1; } 
+     template<> struct switcheroo<ComplexF> {
+       static inline int iscomplex()  { return 1; }
        template<class vec>
        static inline vec mult(vec a, vec b) {
 	 return a*b;
@@ -90,14 +90,14 @@ namespace Grid {
       // Instantiate different versions depending on Impl
       /////////////////////////////////////////////////////
       void M5D(const FermionField &psi,
-	       const FermionField &phi, 
+	       const FermionField &phi,
 	       FermionField &chi,
 	       std::vector<Coeff_t> &lower,
 	       std::vector<Coeff_t> &diag,
 	       std::vector<Coeff_t> &upper);
 
       void M5Ddag(const FermionField &psi,
-		  const FermionField &phi, 
+		  const FermionField &phi,
 		  FermionField &chi,
 		  std::vector<Coeff_t> &lower,
 		  std::vector<Coeff_t> &diag,
@@ -125,7 +125,7 @@ namespace Grid {
 
       // Efficient support for multigrid coarsening
       virtual void  Mdir (const FermionField &in, FermionField &out,int dir,int disp);
-      
+
       void   Meooe5D       (const FermionField &in, FermionField &out);
       void   MeooeDag5D    (const FermionField &in, FermionField &out);
 
@@ -133,23 +133,23 @@ namespace Grid {
       RealD mass;
 
       // Cayley form Moebius (tanh and zolotarev)
-      std::vector<Coeff_t> omega; 
+      std::vector<Coeff_t> omega;
       std::vector<Coeff_t> bs;    // S dependent coeffs
-      std::vector<Coeff_t> cs;    
-      std::vector<Coeff_t> as;    
+      std::vector<Coeff_t> cs;
+      std::vector<Coeff_t> as;
       // For preconditioning Cayley form
-      std::vector<Coeff_t> bee;    
-      std::vector<Coeff_t> cee;    
-      std::vector<Coeff_t> aee;    
-      std::vector<Coeff_t> beo;    
-      std::vector<Coeff_t> ceo;    
-      std::vector<Coeff_t> aeo;    
+      std::vector<Coeff_t> bee;
+      std::vector<Coeff_t> cee;
+      std::vector<Coeff_t> aee;
+      std::vector<Coeff_t> beo;
+      std::vector<Coeff_t> ceo;
+      std::vector<Coeff_t> aeo;
       // LDU factorisation of the eeoo matrix
-      std::vector<Coeff_t> lee;    
-      std::vector<Coeff_t> leem;    
-      std::vector<Coeff_t> uee;    
-      std::vector<Coeff_t> ueem;    
-      std::vector<Coeff_t> dee;    
+      std::vector<Coeff_t> lee;
+      std::vector<Coeff_t> leem;
+      std::vector<Coeff_t> uee;
+      std::vector<Coeff_t> ueem;
+      std::vector<Coeff_t> dee;
 
       // Matrices of 5d ee inverse params
       Vector<iSinglet<Simd> >  MatpInv;
@@ -165,7 +165,7 @@ namespace Grid {
 		      GridRedBlackCartesian &FourDimRedBlackGrid,
 		      RealD _mass,RealD _M5,const ImplParams &p= ImplParams());
 
-      
+
 
      void CayleyReport(void);
      void CayleyZeroCounters(void);
@@ -179,9 +179,9 @@ namespace Grid {
      double MooeeInvTime;
 
     protected:
-      void SetCoefficientsZolotarev(RealD zolohi,Approx::zolotarev_data *zdata,RealD b,RealD c);
-      void SetCoefficientsTanh(Approx::zolotarev_data *zdata,RealD b,RealD c);
-      void SetCoefficientsInternal(RealD zolo_hi,std::vector<Coeff_t> & gamma,RealD b,RealD c);
+      virtual void SetCoefficientsZolotarev(RealD zolohi,Approx::zolotarev_data *zdata,RealD b,RealD c);
+      virtual void SetCoefficientsTanh(Approx::zolotarev_data *zdata,RealD b,RealD c);
+      virtual void SetCoefficientsInternal(RealD zolo_hi,std::vector<Coeff_t> & gamma,RealD b,RealD c);
     };
 
   }

From 6d0786ff9d388d0ee24d2d4087f05db38655b32e Mon Sep 17 00:00:00 2001
From: David Murphy <djm2131@columbia.edu>
Date: Tue, 15 Aug 2017 22:47:00 -0400
Subject: [PATCH 034/115] Typo fixes and check-in of G-parity action test for
 DWF

---
 lib/qcd/action/fermion/AbstractEOFAFermion.h  | 106 ++++-----
 .../action/fermion/DomainWallEOFAFermion.h    | 130 ++++++-----
 tests/debug/Test_reweight_dwf_eofa_gparity.cc | 209 ++++++++++++++++++
 3 files changed, 336 insertions(+), 109 deletions(-)
 create mode 100644 tests/debug/Test_reweight_dwf_eofa_gparity.cc

diff --git a/lib/qcd/action/fermion/AbstractEOFAFermion.h b/lib/qcd/action/fermion/AbstractEOFAFermion.h
index abe06b8c..15faa401 100644
--- a/lib/qcd/action/fermion/AbstractEOFAFermion.h
+++ b/lib/qcd/action/fermion/AbstractEOFAFermion.h
@@ -35,66 +35,66 @@ See the full license in the file "LICENSE" in the top level distribution directo
 namespace Grid {
 namespace QCD {
 
-    // DJM: Abstract base class for EOFA fermion types.
-    // Defines layout of additional EOFA-specific parameters and operators.
-    // Use to construct EOFA pseudofermion actions that are agnostic to Shamir / Mobius / etc.,
-    // and ensure that no one can construct EOFA pseudofermion action with non-EOFA fermion type.
-    template<class Impl>
-    class AbstractEOFAFermion : public CayleyFermion5D<Impl> {
-        public:
-            INHERIT_IMPL_TYPES(Impl);
+  // DJM: Abstract base class for EOFA fermion types.
+  // Defines layout of additional EOFA-specific parameters and operators.
+  // Use to construct EOFA pseudofermion actions that are agnostic to
+  // Shamir / Mobius / etc., and ensure that no one can construct EOFA
+  // pseudofermion action with non-EOFA fermion type.
+  template<class Impl>
+  class AbstractEOFAFermion : public CayleyFermion5D<Impl> {
+    public:
+      INHERIT_IMPL_TYPES(Impl);
 
-        public:
-            // Fermion operator: D(mq1) + shift*\gamma_{5}*R_{5}*\Delta_{\pm}(mq2,mq3)*P_{\pm}
-            RealD mq1;
-            RealD mq2;
-            RealD mq3;
-            RealD shift;
-            int pm;
+    public:
+      // Fermion operator: D(mq1) + shift*\gamma_{5}*R_{5}*\Delta_{\pm}(mq2,mq3)*P_{\pm}
+      RealD mq1;
+      RealD mq2;
+      RealD mq3;
+      RealD shift;
+      int pm;
 
-            RealD alpha; // Mobius scale
-            RealD k;     // EOFA normalization constant
+      RealD alpha; // Mobius scale
+      RealD k;     // EOFA normalization constant
 
-            virtual void Instantiatable(void) = 0;
+      virtual void Instantiatable(void) = 0;
 
-            // EOFA-specific operations
-            // Force user to implement in derived classes
-            virtual void Omega    (const FermionField& in, FermionField& out, int sign, int dag) = 0;
-            virtual void Dtilde   (const FermionField& in, FermionField& out) = 0;
-            virtual void DtildeInv(const FermionField& in, FermionField& out) = 0;
+      // EOFA-specific operations
+      // Force user to implement in derived classes
+      virtual void  Omega    (const FermionField& in, FermionField& out, int sign, int dag) = 0;
+      virtual void  Dtilde   (const FermionField& in, FermionField& out) = 0;
+      virtual void  DtildeInv(const FermionField& in, FermionField& out) = 0;
 
-            // Implement derivatives in base clcass: for EOFA both DWF and Mobius just need d(Dw)/dU
-            virtual void MDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag){
-                this->DhopDeriv(mat, U, V, dag);
-            };
-            virtual void MoeDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag){
-                this->DhopDerivOE(mat, U, V, dag);
-            };
-            virtual void MeoDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag){
-                this->DhopDerivEO(mat, U, V, dag);
-            };
+      // Implement derivatives in base class:
+      // for EOFA both DWF and Mobius just need d(Dw)/dU
+      virtual void MDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag){
+        this->DhopDeriv(mat, U, V, dag);
+      };
+      virtual void MoeDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag){
+        this->DhopDerivOE(mat, U, V, dag);
+      };
+      virtual void MeoDeriv(GaugeField& mat, const FermionField& U, const FermionField& V, int dag){
+        this->DhopDerivEO(mat, U, V, dag);
+      };
 
-            // Recompute 5D coefficients for different value of shift constant (needed for heatbath loop over poles)
-            virtual void RefreshShiftCoefficients(RealD new_shift) = 0;
+      // Recompute 5D coefficients for different value of shift constant
+      // (needed for heatbath loop over poles)
+      virtual void RefreshShiftCoefficients(RealD new_shift) = 0;
 
-            // Constructors
-            AbstractEOFAFermion(GaugeField&            _Umu,
-                                GridCartesian&         FiveDimGrid,
-                                GridRedBlackCartesian& FiveDimRedBlackGrid,
-                                GridCartesian&         FourDimGrid,
-                                GridRedBlackCartesian& FourDimRedBlackGrid,
-                                RealD _mq1, RealD _mq2, RealD _mq3, RealD _shift, int _pm,
-                                RealD _M5, RealD _b, RealD _c, const ImplParams &p= ImplParams()) :
-                                CayleyFermion5D<Impl>(_Umu, FiveDimGrid, FiveDimRedBlackGrid, FourDimGrid, FourDimRedBlackGrid, _mq1, _M5, p),
-                                mq1(_mq1), mq2(_mq2), mq3(_mq3), shift(_shift), pm(_pm)
-            {
-                int Ls = this->Ls;
-                this->alpha = _b + _c;
-                this->k = this->alpha * (_mq3 - _mq2) * std::pow(this->alpha+1.0,2*Ls) /
-                    ( std::pow(alpha+1.0,Ls) + _mq2*std::pow(alpha-1.0,Ls) ) /
-                    ( std::pow(alpha+1.0,Ls) + _mq3*std::pow(alpha-1.0,Ls) );
-            }
-    };
+      // Constructors
+      AbstractEOFAFermion(GaugeField& _Umu, GridCartesian& FiveDimGrid, GridRedBlackCartesian& FiveDimRedBlackGrid,
+        GridCartesian& FourDimGrid, GridRedBlackCartesian& FourDimRedBlackGrid,
+        RealD _mq1, RealD _mq2, RealD _mq3, RealD _shift, int _pm,
+        RealD _M5, RealD _b, RealD _c, const ImplParams& p=ImplParams())
+        : CayleyFermion5D<Impl>(_Umu, FiveDimGrid, FiveDimRedBlackGrid, FourDimGrid, FourDimRedBlackGrid,
+          _mq1, _M5, p), mq1(_mq1), mq2(_mq2), mq3(_mq3), shift(_shift), pm(_pm)
+      {
+        int Ls = this->Ls;
+        this->alpha = _b + _c;
+        this->k = this->alpha * (_mq3-_mq2) * std::pow(this->alpha+1.0,2*Ls) /
+                    ( std::pow(this->alpha+1.0,Ls) + _mq2*std::pow(this->alpha-1.0,Ls) ) /
+                    ( std::pow(this->alpha+1.0,Ls) + _mq3*std::pow(this->alpha-1.0,Ls) );
+      };
+  };
 }}
 
 #endif
diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermion.h b/lib/qcd/action/fermion/DomainWallEOFAFermion.h
index 179736ba..d48e3b8f 100644
--- a/lib/qcd/action/fermion/DomainWallEOFAFermion.h
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermion.h
@@ -35,75 +35,93 @@ See the full license in the file "LICENSE" in the top level distribution directo
 namespace Grid {
 namespace QCD {
 
-    // DJM: EOFA with (Shamir) domain wall fermions.
-    // We overload and re-implement only the routines which have a different operator
-    // structure than the CayleyFermion5D base class.
-    template<class Impl>
-    class DomainWallEOFAFermion : public AbstractEOFAFermion<Impl>
-    {
-        public:
-          INHERIT_IMPL_TYPES(Impl);
+  /*template<typename T> struct switcheroo {
+    static inline int iscomplex() { return 0; }
+    template<class vec>
+    static inline vec mult(vec a, vec b){ return real_mult(a,b); }
+  };
 
-        public:
-          // Modified (0,Ls-1) and (Ls-1,0) elements of Mooee for red-black preconditioned Shamir EOFA
-          Coeff_t dm;
-          Coeff_t dp;
+  template<> struct switcheroo<ComplexD> {
+    static inline int iscomplex() { return 1; }
+    template<class vec>
+    static inline vec mult(vec a, vec b){ return a*b; }
+  };
 
-          virtual void Instantiatable(void) {};
+  template<> struct switcheroo<ComplexF> {
+    static inline int iscomplex() { return 1; }
+    template<class vec>
+    static inline vec mult(vec a, vec b) { return a*b; }
+  };*/
 
-          // EOFA specific operators
-          virtual void   Omega      (const FermionField& in, FermionField &out, int sign, int dag);
-          virtual void   Dtilde     (const FermionField& in, FermionField &out);
-          virtual void   DtildeInv  (const FermionField& in, FermionField &out);
+  template<class Impl>
+  class DomainWallEOFAFermion : public AbstractEOFAFermion<Impl>
+  {
+    public:
+      INHERIT_IMPL_TYPES(Impl);
 
-          // override multiply
-          virtual RealD  M          (const FermionField& in, FermionField& out);
-          virtual RealD  Mdag       (const FermionField& in, FermionField& out);
+    public:
+      // Modified (0,Ls-1) and (Ls-1,0) elements of Mooee
+      // for red-black preconditioned Shamir EOFA
+      Coeff_t dm;
+      Coeff_t dp;
 
-          // half checkerboard operations
-          virtual void   Mooee      (const FermionField &in, FermionField &out);
-          virtual void   MooeeDag   (const FermionField &in, FermionField &out);
-          virtual void   MooeeInv   (const FermionField &in, FermionField &out);
-          virtual void   MooeeInvDag(const FermionField &in, FermionField &out);
+      virtual void Instantiatable(void) {};
 
-          virtual void   M5D        (const FermionField &psi, FermionField &chi);
-          virtual void   M5Ddag     (const FermionField &psi, FermionField &chi);
+      // EOFA-specific operations
+      virtual void  Omega      (const FermionField& in, FermionField& out, int sign, int dag);
+      virtual void  Dtilde     (const FermionField& in, FermionField& out);
+      virtual void  DtildeInv  (const FermionField& in, FermionField& out);
 
-          /////////////////////////////////////////////////////
-          // Instantiate different versions depending on Impl
-          /////////////////////////////////////////////////////
-          void M5D                 (const FermionField &psi, const FermionField &phi, FermionField &chi,
-                                        std::vector<Coeff_t> &lower, std::vector<Coeff_t> &diag, std::vector<Coeff_t> &upper);
-          void M5Ddag              (const FermionField &psi, const FermionField &phi, FermionField &chi,
-                                        std::vector<Coeff_t> &lower, std::vector<Coeff_t> &diag, std::vector<Coeff_t> &upper);
-          void MooeeInternal       (const FermionField &in, FermionField &out, int dag, int inv);
-          void MooeeInternalCompute(int dag, int inv, Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm);
-          void MooeeInternalAsm    (const FermionField &in, FermionField &out, int LLs, int site,
-                                        Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm);
-          void MooeeInternalZAsm   (const FermionField &in, FermionField &out, int LLs, int site,
-                                        Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm);
+      // override multiply
+      virtual RealD M          (const FermionField& in, FermionField& out);
+      virtual RealD Mdag       (const FermionField& in, FermionField& out);
 
-          virtual void RefreshShiftCoefficients(RealD new_shift);
+      // half checkerboard operations
+      virtual void  Mooee      (const FermionField& in, FermionField& out);
+      virtual void  MooeeDag   (const FermionField& in, FermionField& out);
+      virtual void  MooeeInv   (const FermionField& in, FermionField& out);
+      virtual void  MooeeInvDag(const FermionField& in, FermionField& out);
 
-          // Constructors
-          DomainWallEOFAFermion(GaugeField&            _Umu,
-                                GridCartesian&         FiveDimGrid,
-                                GridRedBlackCartesian& FiveDimRedBlackGrid,
-                                GridCartesian&         FourDimGrid,
-                                GridRedBlackCartesian& FourDimRedBlackGrid,
-                                RealD _mq1, RealD _mq2, RealD _mq3, RealD _shift,
-                                int pm, RealD _M5, const ImplParams& p=ImplParams());
+      virtual void   M5D       (const FermionField& psi, FermionField& chi);
+      virtual void   M5Ddag    (const FermionField& psi, FermionField& chi);
 
-        protected:
-          virtual void SetCoefficientsInternal(RealD zolo_hi, std::vector<Coeff_t> &gamma, RealD b, RealD c);
-    };
+      /////////////////////////////////////////////////////
+      // Instantiate different versions depending on Impl
+      /////////////////////////////////////////////////////
+      void M5D(const FermionField& psi, const FermionField& phi, FermionField& chi,
+        std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper);
+
+      void M5Ddag(const FermionField& psi, const FermionField& phi, FermionField& chi,
+        std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper);
+
+      void MooeeInternal(const FermionField& in, FermionField& out, int dag, int inv);
+
+      void MooeeInternalCompute(int dag, int inv, Vector<iSinglet<Simd>>& Matp, Vector<iSinglet<Simd>>& Matm);
+
+      void MooeeInternalAsm(const FermionField& in, FermionField& out, int LLs, int site,
+        Vector<iSinglet<Simd>>& Matp, Vector<iSinglet<Simd>>& Matm);
+
+      void MooeeInternalZAsm(const FermionField& in, FermionField& out, int LLs, int site,
+        Vector<iSinglet<Simd>>& Matp, Vector<iSinglet<Simd>>& Matm);
+
+      virtual void RefreshShiftCoefficients(RealD new_shift);
+
+      // Constructors
+      DomainWallEOFAFermion(GaugeField& _Umu, GridCartesian& FiveDimGrid, GridRedBlackCartesian& FiveDimRedBlackGrid,
+        GridCartesian& FourDimGrid, GridRedBlackCartesian& FourDimRedBlackGrid,
+        RealD _mq1, RealD _mq2, RealD _mq3, RealD _shift, int pm,
+        RealD _M5, const ImplParams& p=ImplParams());
+
+    protected:
+      void SetCoefficientsInternal(RealD zolo_hi, std::vector<Coeff_t>& gamma, RealD b, RealD c);
+  };
 }}
 
 #define INSTANTIATE_DPERP_DWF_EOFA(A)\
-template void DomainWallEOFAFermion<A>::M5D(const FermionField& psi, const FermionField& phi, FermionField& chi,\
-    std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper); \
-template void DomainWallEOFAFermion<A>::M5Ddag(const FermionField& psi, const FermionField& phi, FermionField& chi,\
-    std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper); \
+template void DomainWallEOFAFermion<A>::M5D(const FermionField& psi, const FermionField& phi, FermionField& chi, \
+  std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper); \
+template void DomainWallEOFAFermion<A>::M5Ddag(const FermionField& psi, const FermionField& phi, FermionField& chi, \
+  std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper); \
 template void DomainWallEOFAFermion<A>::MooeeInv(const FermionField& psi, FermionField& chi); \
 template void DomainWallEOFAFermion<A>::MooeeInvDag(const FermionField& psi, FermionField& chi);
 
diff --git a/tests/debug/Test_reweight_dwf_eofa_gparity.cc b/tests/debug/Test_reweight_dwf_eofa_gparity.cc
new file mode 100644
index 00000000..b77ec33e
--- /dev/null
+++ b/tests/debug/Test_reweight_dwf_eofa_gparity.cc
@@ -0,0 +1,209 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/debug/Test_reweight_dwf_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+typedef typename GparityDomainWallFermionR::FermionField FermionField;
+
+// parameters for test
+const std::vector<int> grid_dim = { 8, 8, 8, 8 };
+const int Ls = 8;
+const int Nhits = 10;
+const int max_iter = 5000;
+const RealD mf = 0.1;
+const RealD mb = 0.11;
+const RealD M5 = 1.8;
+const RealD stop_tol = 1.0e-12;
+
+RealD mean(const std::vector<RealD>& data)
+{
+    int N = data.size();
+    RealD mean(0.0);
+    for(int i=0; i<N; ++i){ mean += data[i]; }
+    return mean/RealD(N);
+}
+
+RealD jack_mean(const std::vector<RealD>& data, int sample)
+{
+    int N = data.size();
+    RealD mean(0.0);
+    for(int i=0; i<N; ++i){ if(i != sample){ mean += data[i]; } }
+    return mean/RealD(N-1);
+}
+
+RealD jack_std(const std::vector<RealD>& jacks, RealD mean)
+{
+    int N = jacks.size();
+    RealD std(0.0);
+    for(int i=0; i<N; ++i){ std += std::pow(jacks[i]-mean, 2.0); }
+    return std::sqrt(RealD(N-1)/RealD(N)*std);
+}
+
+std::vector<RealD> jack_stats(const std::vector<RealD>& data)
+{
+    int N = data.size();
+    std::vector<RealD> jack_samples(N);
+    std::vector<RealD> jack_stats(2);
+
+    jack_stats[0] = mean(data);
+    for(int i=0; i<N; i++){ jack_samples[i] = jack_mean(data,i); }
+    jack_stats[1] = jack_std(jack_samples, jack_stats[0]);
+    return jack_stats;
+}
+
+int main(int argc, char **argv)
+{
+  Grid_init(&argc, &argv);
+
+  // Initialize spacetime grid
+  std::cout << GridLogMessage << "Lattice dimensions: "
+    << grid_dim << "   Ls: " << Ls << std::endl;
+  GridCartesian* UGrid = SpaceTimeGrid::makeFourDimGrid(grid_dim,
+      GridDefaultSimd(Nd, vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian* FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Set up RNGs
+  std::vector<int> seeds4({1, 2, 3, 4});
+  std::vector<int> seeds5({5, 6, 7, 8});
+  GridParallelRNG RNG5(FGrid);
+  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);
+  RNG4.SeedFixedIntegers(seeds4);
+
+  // Random gauge field
+  LatticeGaugeField Umu(UGrid);
+  SU3::HotConfiguration(RNG4, Umu);
+
+  // Initialize RHMC fermion operators
+  GparityDomainWallFermionR::ImplParams params;
+  GparityDomainWallFermionR Ddwf_f(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, M5, params);
+  GparityDomainWallFermionR Ddwf_b(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, M5, params);
+  SchurDiagMooeeOperator<GparityDomainWallFermionR, FermionField> MdagM(Ddwf_f);
+  SchurDiagMooeeOperator<GparityDomainWallFermionR, FermionField> VdagV(Ddwf_b);
+
+  // Degree 12 rational approximations to x^(1/4) and x^(-1/4)
+  double     lo = 0.0001;
+  double     hi = 95.0;
+  int precision = 64;
+  int    degree = 12;
+  AlgRemez remez(lo, hi, precision);
+  std::cout << GridLogMessage << "Generating degree " << degree << " for x^(1/4)" << std::endl;
+  remez.generateApprox(degree, 1, 4);
+  MultiShiftFunction PowerQuarter(remez, stop_tol, false);
+  MultiShiftFunction PowerNegQuarter(remez, stop_tol, true);
+
+  // Stochastically estimate reweighting factor via RHMC
+  RealD scale = std::sqrt(0.5);
+  std::vector<RealD> rw_rhmc(Nhits);
+  ConjugateGradientMultiShift<FermionField> msCG_V(max_iter, PowerQuarter);
+  ConjugateGradientMultiShift<FermionField> msCG_M(max_iter, PowerNegQuarter);
+  std::cout.precision(12);
+
+  for(int hit=0; hit<Nhits; hit++){
+
+    // Gaussian source
+    FermionField Phi    (Ddwf_f.FermionGrid());
+    FermionField PhiOdd (Ddwf_f.FermionRedBlackGrid());
+    std::vector<FermionField> tmp(2, Ddwf_f.FermionRedBlackGrid());
+    gaussian(RNG5, Phi);
+    Phi = Phi*scale;
+
+    pickCheckerboard(Odd, PhiOdd, Phi);
+
+    // evaluate -log(rw)
+    msCG_V(VdagV, PhiOdd, tmp[0]);
+    msCG_M(MdagM, tmp[0], tmp[1]);
+    rw_rhmc[hit] = norm2(tmp[1]) - norm2(PhiOdd);
+    std::cout << std::endl << "==================================================" << std::endl;
+    std::cout << " --- RHMC: Hit " << hit << ": rw = " << rw_rhmc[hit];
+    std::cout << std::endl << "==================================================" << std::endl << std::endl;
+
+  }
+
+  // Initialize EOFA fermion operators
+  RealD shift_L = 0.0;
+  RealD shift_R = -1.0;
+  int pm = 1;
+  GparityDomainWallEOFAFermionR Deofa_L(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, mf, mb, shift_L, pm, M5, params);
+  GparityDomainWallEOFAFermionR Deofa_R(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, mf, mb, shift_R, pm, M5, params);
+  MdagMLinearOperator<GparityDomainWallEOFAFermionR, FermionField> LdagL(Deofa_L);
+  MdagMLinearOperator<GparityDomainWallEOFAFermionR, FermionField> RdagR(Deofa_R);
+
+  // Stochastically estimate reweighting factor via EOFA
+  RealD k = Deofa_L.k;
+  std::vector<RealD> rw_eofa(Nhits);
+  ConjugateGradient<FermionField> CG(stop_tol, max_iter);
+  SchurRedBlackDiagMooeeSolve<FermionField> SchurSolver(CG);
+
+  for(int hit=0; hit<Nhits; hit++){
+
+    // Gaussian source
+    FermionField Phi       (Deofa_L.FermionGrid());
+    FermionField spProj_Phi(Deofa_L.FermionGrid());
+    std::vector<FermionField> tmp(2, Deofa_L.FermionGrid());
+    gaussian(RNG5, Phi);
+    Phi = Phi*scale;
+
+    // evaluate -log(rw)
+    // LH term
+    for(int s=0; s<Ls; ++s){ axpby_ssp_pminus(spProj_Phi, 0.0, Phi, 1.0, Phi, s, s); }
+    Deofa_L.Omega(spProj_Phi, tmp[0], -1, 0);
+    G5R5(tmp[1], tmp[0]);
+    tmp[0] = zero;
+    SchurSolver(Deofa_L, tmp[1], tmp[0]);
+    Deofa_L.Omega(tmp[0], tmp[1], -1, 1);
+    rw_eofa[hit] = -k*innerProduct(spProj_Phi,tmp[1]).real();
+
+    // RH term
+    for(int s=0; s<Ls; ++s){ axpby_ssp_pplus(spProj_Phi, 0.0, Phi, 1.0, Phi, s, s); }
+    Deofa_R.Omega(spProj_Phi, tmp[0], 1, 0);
+    G5R5(tmp[1], tmp[0]);
+    tmp[0] = zero;
+    SchurSolver(Deofa_R, tmp[1], tmp[0]);
+    Deofa_R.Omega(tmp[0], tmp[1], 1, 1);
+    rw_eofa[hit] += k*innerProduct(spProj_Phi,tmp[1]).real();
+    std::cout << std::endl << "==================================================" << std::endl;
+    std::cout << " --- EOFA: Hit " << hit << ": rw = " << rw_eofa[hit];
+    std::cout << std::endl << "==================================================" << std::endl << std::endl;
+
+  }
+
+  std::vector<RealD> rhmc_result = jack_stats(rw_rhmc);
+  std::vector<RealD> eofa_result = jack_stats(rw_eofa);
+  std::cout << std::endl << "RHMC: rw = " << rhmc_result[0] << " +/- " << rhmc_result[1] << std::endl;
+  std::cout << std::endl << "EOFA: rw = " << eofa_result[0] << " +/- " << eofa_result[1] << std::endl;
+
+  Grid_finalize();
+}

From fd367d8bfd95ec193b9528c59d7846508bf82296 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Wed, 16 Aug 2017 09:42:57 +0100
Subject: [PATCH 035/115] Debugging the PointerCache

---
 lib/allocator/AlignedAllocator.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/allocator/AlignedAllocator.cc b/lib/allocator/AlignedAllocator.cc
index 4249a72e..04de20bf 100644
--- a/lib/allocator/AlignedAllocator.cc
+++ b/lib/allocator/AlignedAllocator.cc
@@ -11,7 +11,7 @@ int PointerCache::victim;
 
 void *PointerCache::Insert(void *ptr,size_t bytes) {
 
-  if (bytes < 4096 ) return NULL;
+  if (bytes < 4096 ) return ptr;
 
 #ifdef GRID_OMP
   assert(omp_in_parallel()==0);

From 41f73ec0836fd5ec3093edba4174b466815cf799 Mon Sep 17 00:00:00 2001
From: David Murphy <djm2131@columbia.edu>
Date: Wed, 16 Aug 2017 12:37:38 -0400
Subject: [PATCH 036/115] Add ChronoForecast class for forecasting solutions
 across poles in the EOFA heatbath

---
 lib/algorithms/Algorithms.h      |   3 +-
 lib/algorithms/approx/Forecast.h | 152 +++++++++++++++++++++++++++++++
 2 files changed, 154 insertions(+), 1 deletion(-)
 create mode 100644 lib/algorithms/approx/Forecast.h

diff --git a/lib/algorithms/Algorithms.h b/lib/algorithms/Algorithms.h
index 5123c7a1..361ccd9c 100644
--- a/lib/algorithms/Algorithms.h
+++ b/lib/algorithms/Algorithms.h
@@ -1,6 +1,6 @@
     /*************************************************************************************
 
-    Grid physics library, www.github.com/paboyle/Grid 
+    Grid physics library, www.github.com/paboyle/Grid
 
     Source file: ./lib/Algorithms.h
 
@@ -37,6 +37,7 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <Grid/algorithms/approx/Chebyshev.h>
 #include <Grid/algorithms/approx/Remez.h>
 #include <Grid/algorithms/approx/MultiShiftFunction.h>
+#include <Grid/algorithms/approx/Forecast.h>
 
 #include <Grid/algorithms/iterative/ConjugateGradient.h>
 #include <Grid/algorithms/iterative/ConjugateResidual.h>
diff --git a/lib/algorithms/approx/Forecast.h b/lib/algorithms/approx/Forecast.h
new file mode 100644
index 00000000..87eb84a6
--- /dev/null
+++ b/lib/algorithms/approx/Forecast.h
@@ -0,0 +1,152 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/algorithms/approx/Forecast.h
+
+Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#ifndef INCLUDED_FORECAST_H
+#define INCLUDED_FORECAST_H
+
+namespace Grid {
+
+  // Abstract base class.
+  // Takes a matrix (Mat), a source (phi), and a vector of Fields (chi)
+  // and returns a forecasted solution to the system D*psi = phi (psi).
+  template<class Matrix, class Field>
+  class Forecast
+  {
+    public:
+      virtual Field operator()(Matrix &Mat, const Field& phi, const std::vector<Field>& chi) = 0;
+  };
+
+  // Implementation of Brower et al.'s chronological inverter (arXiv:hep-lat/9509012),
+  // used to forecast solutions across poles of the EOFA heatbath.
+  //
+  // Modified from CPS (cps_pp/src/util/dirac_op/d_op_base/comsrc/minresext.C)
+  template<class Matrix, class Field>
+  class ChronoForecast : public Forecast<Matrix,Field>
+  {
+    public:
+      Field operator()(Matrix &Mat, const Field& phi, const std::vector<Field>& prev_solns)
+      {
+        int degree = prev_solns.size();
+        Field chi(phi); // forecasted solution
+
+        // Trivial cases
+        if(degree == 0){ chi = zero; return chi; }
+        else if(degree == 1){ return prev_solns[0]; }
+
+        RealD dot;
+        ComplexD xp;
+        Field r(phi); // residual
+        Field Mv(phi);
+        std::vector<Field> v(prev_solns); // orthonormalized previous solutions
+        std::vector<Field> MdagMv(degree,phi);
+
+        // Array to hold the matrix elements
+        std::vector<std::vector<ComplexD>> G(degree, std::vector<ComplexD>(degree));
+
+        // Solution and source vectors
+        std::vector<ComplexD> a(degree);
+        std::vector<ComplexD> b(degree);
+
+        // Orthonormalize the vector basis
+        for(int i=0; i<degree; i++){
+          v[i] *= 1.0/std::sqrt(norm2(v[i]));
+          for(int j=i+1; j<degree; j++){ v[j] -= innerProduct(v[i],v[j]) * v[i]; }
+        }
+
+        // Perform sparse matrix multiplication and construct rhs
+        for(int i=0; i<degree; i++){
+          b[i] = innerProduct(v[i],phi);
+          Mat.M(v[i],Mv);
+          Mat.Mdag(Mv,MdagMv[i]);
+          G[i][i] = innerProduct(v[i],MdagMv[i]);
+        }
+
+        // Construct the matrix
+        for(int j=0; j<degree; j++){
+        for(int k=j+1; k<degree; k++){
+          G[j][k] = innerProduct(v[j],MdagMv[k]);
+          G[k][j] = std::conj(G[j][k]);
+        }}
+
+        // Gauss-Jordan elimination with partial pivoting
+        for(int i=0; i<degree; i++){
+
+          // Perform partial pivoting
+          int k = i;
+          for(int j=i+1; j<degree; j++){ if(std::abs(G[j][j]) > std::abs(G[k][k])){ k = j; } }
+          if(k != i){
+            xp = b[k];
+            b[k] = b[i];
+            b[i] = xp;
+            for(int j=0; j<degree; j++){
+              xp = G[k][j];
+              G[k][j] = G[i][j];
+              G[i][j] = xp;
+            }
+          }
+
+          // Convert matrix to upper triangular form
+          for(int j=i+1; j<degree; j++){
+            xp = G[j][i]/G[i][i];
+            b[j] -= xp * b[i];
+            for(int k=0; k<degree; k++){ G[j][k] -= xp*G[i][k]; }
+          }
+        }
+
+        // Use Gaussian elimination to solve equations and calculate initial guess
+        chi = zero;
+        r = phi;
+        for(int i=degree-1; i>=0; i--){
+          a[i] = 0.0;
+          for(int j=i+1; j<degree; j++){ a[i] += G[i][j] * a[j]; }
+          a[i] = (b[i]-a[i])/G[i][i];
+          chi += a[i]*v[i];
+          r -= a[i]*MdagMv[i];
+        }
+
+        RealD true_r(0.0);
+        ComplexD tmp;
+        for(int i=0; i<degree; i++){
+          tmp = -b[i];
+          for(int j=0; j<degree; j++){ tmp += G[i][j]*a[j]; }
+          tmp = std::conj(tmp)*tmp;
+          true_r += std::sqrt(tmp.real());
+        }
+
+        RealD error = std::sqrt(norm2(r)/norm2(phi));
+        std::cout << GridLogMessage << "ChronoForecast: |res|/|src| = " << error << std::endl;
+
+        return chi;
+      };
+  };
+
+}
+
+#endif

From ec1e2f7a40a05f2e0e114fbcee9c6806549daf52 Mon Sep 17 00:00:00 2001
From: David Murphy <djm2131@columbia.edu>
Date: Wed, 16 Aug 2017 12:38:59 -0400
Subject: [PATCH 037/115] Add (mostly implemented) ExactOneFlavourRatio
 pseudofermion class and tests of Shamir heatbath and action

---
 .../pseudofermion/ExactOneFlavourRatio.h      | 230 ++++++++++++++++++
 lib/qcd/action/pseudofermion/PseudoFermion.h  |   1 +
 tests/debug/Test_heatbath_dwf_eofa.cc         | 102 ++++++++
 tests/debug/Test_heatbath_dwf_eofa_gparity.cc | 108 ++++++++
 4 files changed, 441 insertions(+)
 create mode 100644 lib/qcd/action/pseudofermion/ExactOneFlavourRatio.h
 create mode 100644 tests/debug/Test_heatbath_dwf_eofa.cc
 create mode 100644 tests/debug/Test_heatbath_dwf_eofa_gparity.cc

diff --git a/lib/qcd/action/pseudofermion/ExactOneFlavourRatio.h b/lib/qcd/action/pseudofermion/ExactOneFlavourRatio.h
new file mode 100644
index 00000000..38c7380c
--- /dev/null
+++ b/lib/qcd/action/pseudofermion/ExactOneFlavourRatio.h
@@ -0,0 +1,230 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/pseudofermion/ExactOneFlavourRatio.h
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+/////////////////////////////////////////////////////////////////
+// Implementation of exact one flavour algorithm (EOFA)         //
+// using fermion classes defined in:                           //
+//    Grid/qcd/action/fermion/DomainWallEOFAFermion.h (Shamir) //
+//    Grid/qcd/action/fermion/MobiusEOFAFermion.h (Mobius)     //
+// arXiv: 1403.1683, 1706.05843                                //
+/////////////////////////////////////////////////////////////////
+
+#ifndef QCD_PSEUDOFERMION_EXACT_ONE_FLAVOUR_RATIO_H
+#define QCD_PSEUDOFERMION_EXACT_ONE_FLAVOUR_RATIO_H
+
+namespace Grid{
+namespace QCD{
+
+  ///////////////////////////////////////////////////////////////
+  // Exact one flavour implementation of DWF determinant ratio //
+  ///////////////////////////////////////////////////////////////
+
+  template<class Impl>
+  class ExactOneFlavourRatioPseudoFermionAction : public Action<typename Impl::GaugeField>
+  {
+    public:
+      INHERIT_IMPL_TYPES(Impl);
+      typedef OneFlavourRationalParams Params;
+      Params param;
+      MultiShiftFunction PowerNegHalf;
+
+    private:
+      bool use_heatbath_forecasting;
+      AbstractEOFAFermion<Impl>& Lop; // the basic LH operator
+      AbstractEOFAFermion<Impl>& Rop; // the basic RH operator
+      SchurRedBlackDiagMooeeSolve<FermionField> Solver;
+      FermionField Phi; // the pseudofermion field for this trajectory
+
+    public:
+      ExactOneFlavourRatioPseudoFermionAction(AbstractEOFAFermion<Impl>& _Lop, AbstractEOFAFermion<Impl>& _Rop,
+        OperatorFunction<FermionField>& S, Params& p, bool use_fc=false) : Lop(_Lop), Rop(_Rop), Solver(S),
+        Phi(_Lop.FermionGrid()), param(p), use_heatbath_forecasting(use_fc)
+      {
+        AlgRemez remez(param.lo, param.hi, param.precision);
+
+        // MdagM^(+- 1/2)
+        std::cout << GridLogMessage << "Generating degree " << param.degree << " for x^(-1/2)" << std::endl;
+        remez.generateApprox(param.degree, 1, 2);
+        PowerNegHalf.Init(remez, param.tolerance, true);
+      };
+
+      virtual std::string action_name() { return "ExactOneFlavourRatioPseudoFermionAction"; }
+
+      virtual std::string LogParameters() {
+        std::stringstream sstream;
+        sstream << GridLogMessage << "[" << action_name() << "] Low            :" << param.lo << std::endl;
+        sstream << GridLogMessage << "[" << action_name() << "] High           :" << param.hi << std::endl;
+        sstream << GridLogMessage << "[" << action_name() << "] Max iterations :" << param.MaxIter << std::endl;
+        sstream << GridLogMessage << "[" << action_name() << "] Tolerance      :" << param.tolerance << std::endl;
+        sstream << GridLogMessage << "[" << action_name() << "] Degree         :" << param.degree << std::endl;
+        sstream << GridLogMessage << "[" << action_name() << "] Precision      :" << param.precision << std::endl;
+        return sstream.str();
+      }
+
+      // Spin projection
+      void spProj(const FermionField& in, FermionField& out, int sign, int Ls)
+      {
+        if(sign == 1){ for(int s=0; s<Ls; ++s){ axpby_ssp_pplus(out, 0.0, in, 1.0, in, s, s); } }
+        else{ for(int s=0; s<Ls; ++s){ axpby_ssp_pminus(out, 0.0, in, 1.0, in, s, s); } }
+      }
+
+      // EOFA heatbath: see Eqn. (29) of arXiv:1706.05843
+      // We generate a Gaussian noise vector \eta, and then compute
+      //  \Phi = M_{\rm EOFA}^{-1/2} * \eta
+      // using a rational approximation to the inverse square root
+      virtual void refresh(const GaugeField& U, GridParallelRNG& pRNG)
+      {
+        Lop.ImportGauge(U);
+        Rop.ImportGauge(U);
+
+        FermionField eta         (Lop.FermionGrid());
+        FermionField CG_src      (Lop.FermionGrid());
+        FermionField CG_soln     (Lop.FermionGrid());
+        FermionField Forecast_src(Lop.FermionGrid());
+        std::vector<FermionField> tmp(2, Lop.FermionGrid());
+
+        // Use chronological inverter to forecast solutions across poles
+        std::vector<FermionField> prev_solns;
+        if(use_heatbath_forecasting){ prev_solns.reserve(param.degree); }
+        ChronoForecast<AbstractEOFAFermion<Impl>, FermionField> Forecast;
+
+        // Seed with Gaussian noise vector (var = 0.5)
+        RealD scale = std::sqrt(0.5);
+        gaussian(pRNG,eta);
+        eta = eta * scale;
+        printf("Heatbath source vector: <\eta|\eta> = %1.15e\n", norm2(eta));
+
+        // \Phi = ( \alpha_{0} + \sum_{k=1}^{N_{p}} \alpha_{l} * \gamma_{l} ) * \eta
+        RealD N(PowerNegHalf.norm);
+        for(int k=0; k<param.degree; ++k){ N += PowerNegHalf.residues[k] / ( 1.0 + PowerNegHalf.poles[k] ); }
+        Phi = eta * N;
+
+        // LH terms:
+        // \Phi = \Phi + k \sum_{k=1}^{N_{p}} P_{-} \Omega_{-}^{\dagger} ( H(mf)
+        //          - \gamma_{l} \Delta_{-}(mf,mb) P_{-} )^{-1} \Omega_{-} P_{-} \eta
+        RealD gamma_l(0.0);
+        spProj(eta, tmp[0], -1, Lop.Ls);
+        Lop.Omega(tmp[0], tmp[1], -1, 0);
+        G5R5(CG_src, tmp[1]);
+        tmp[1] = zero;
+        for(int k=0; k<param.degree; ++k){
+          gamma_l = 1.0 / ( 1.0 + PowerNegHalf.poles[k] );
+          Lop.RefreshShiftCoefficients(-gamma_l);
+          if(use_heatbath_forecasting){ // Forecast CG guess using solutions from previous poles
+            Lop.Mdag(CG_src, Forecast_src);
+            CG_soln = Forecast(Lop, Forecast_src, prev_solns);
+            Solver(Lop, CG_src, CG_soln);
+            prev_solns.push_back(CG_soln);
+          } else {
+            CG_soln = zero; // Just use zero as the initial guess
+            Solver(Lop, CG_src, CG_soln);
+          }
+          Lop.Dtilde(CG_soln, tmp[0]); // We actually solved Cayley preconditioned system: transform back
+          tmp[1] = tmp[1] + ( PowerNegHalf.residues[k]*gamma_l*gamma_l*Lop.k ) * tmp[0];
+        }
+        Lop.Omega(tmp[1], tmp[0], -1, 1);
+        spProj(tmp[0], tmp[1], -1, Lop.Ls);
+        Phi = Phi + tmp[1];
+
+        // RH terms:
+        // \Phi = \Phi - k \sum_{k=1}^{N_{p}} P_{+} \Omega_{+}^{\dagger} ( H(mb)
+        //          + \gamma_{l} \Delta_{+}(mf,mb) P_{+} )^{-1} \Omega_{+} P_{+} \eta
+        spProj(eta, tmp[0], 1, Rop.Ls);
+        Rop.Omega(tmp[0], tmp[1], 1, 0);
+        G5R5(CG_src, tmp[1]);
+        tmp[1] = zero;
+        if(use_heatbath_forecasting){ prev_solns.clear(); } // empirically, LH solns don't help for RH solves
+        for(int k=0; k<param.degree; ++k){
+          gamma_l = 1.0 / ( 1.0 + PowerNegHalf.poles[k] );
+          Rop.RefreshShiftCoefficients(-gamma_l*PowerNegHalf.poles[k]);
+          if(use_heatbath_forecasting){
+            Rop.Mdag(CG_src, Forecast_src);
+            CG_soln = Forecast(Rop, Forecast_src, prev_solns);
+            Solver(Rop, CG_src, CG_soln);
+            prev_solns.push_back(CG_soln);
+          } else {
+            CG_soln = zero;
+            Solver(Rop, CG_src, CG_soln);
+          }
+          Rop.Dtilde(CG_soln, tmp[0]); // We actually solved Cayley preconditioned system: transform back
+          tmp[1] = tmp[1] - ( PowerNegHalf.residues[k]*gamma_l*gamma_l*Rop.k ) * tmp[0];
+        }
+        Rop.Omega(tmp[1], tmp[0], 1, 1);
+        spProj(tmp[0], tmp[1], 1, Rop.Ls);
+        Phi = Phi + tmp[1];
+
+        // Reset shift coefficients for energy and force evals
+        Lop.RefreshShiftCoefficients(0.0);
+        Rop.RefreshShiftCoefficients(-1.0);
+      };
+
+      // EOFA action: see Eqn. (10) of arXiv:1706.05843
+      virtual RealD S(const GaugeField& U)
+      {
+        Lop.ImportGauge(U);
+        Rop.ImportGauge(U);
+
+        FermionField spProj_Phi(Lop.FermionGrid());
+        std::vector<FermionField> tmp(2, Lop.FermionGrid());
+
+        // S = <\Phi|\Phi>
+        RealD action(norm2(Phi));
+
+        // LH term: S = S - k <\Phi| P_{-} \Omega_{-}^{\dagger} H(mf)^{-1} \Omega_{-} P_{-} |\Phi>
+        spProj(Phi, spProj_Phi, -1, Lop.Ls);
+        Lop.Omega(spProj_Phi, tmp[0], -1, 0);
+        G5R5(tmp[1], tmp[0]);
+        tmp[0] = zero;
+        Solver(Lop, tmp[1], tmp[0]);
+        Lop.Dtilde(tmp[0], tmp[1]); // We actually solved Cayley preconditioned system: transform back
+        Lop.Omega(tmp[1], tmp[0], -1, 1);
+        action -= Lop.k * innerProduct(spProj_Phi, tmp[0]).real();
+
+        // RH term: S = S + k <\Phi| P_{+} \Omega_{+}^{\dagger} ( H(mb)
+        //               - \Delta_{+}(mf,mb) P_{+} )^{-1} \Omega_{-} P_{-} |\Phi>
+        spProj(Phi, spProj_Phi, 1, Rop.Ls);
+        Rop.Omega(spProj_Phi, tmp[0], 1, 0);
+        G5R5(tmp[1], tmp[0]);
+        tmp[0] = zero;
+        Solver(Rop, tmp[1], tmp[0]);
+        Rop.Dtilde(tmp[0], tmp[1]);
+        Rop.Omega(tmp[1], tmp[0], 1, 1);
+        action += Rop.k * innerProduct(spProj_Phi, tmp[0]).real();
+
+        return action;
+      };
+
+      // EOFA pseudofermion force: see Eqns. (34)-(36) of arXiv:1706.05843
+      virtual void deriv(const GaugeField& U, GaugeField& dSdU)
+      {
+      };
+  };
+}}
+
+#endif
diff --git a/lib/qcd/action/pseudofermion/PseudoFermion.h b/lib/qcd/action/pseudofermion/PseudoFermion.h
index bccca3d4..133ebb7d 100644
--- a/lib/qcd/action/pseudofermion/PseudoFermion.h
+++ b/lib/qcd/action/pseudofermion/PseudoFermion.h
@@ -38,5 +38,6 @@ directory
 #include <Grid/qcd/action/pseudofermion/OneFlavourRationalRatio.h>
 #include <Grid/qcd/action/pseudofermion/OneFlavourEvenOddRational.h>
 #include <Grid/qcd/action/pseudofermion/OneFlavourEvenOddRationalRatio.h>
+#include <Grid/qcd/action/pseudofermion/ExactOneFlavourRatio.h>
 
 #endif
diff --git a/tests/debug/Test_heatbath_dwf_eofa.cc b/tests/debug/Test_heatbath_dwf_eofa.cc
new file mode 100644
index 00000000..b77bc982
--- /dev/null
+++ b/tests/debug/Test_heatbath_dwf_eofa.cc
@@ -0,0 +1,102 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./tests/debug/Test_heatbath_dwf_eofa.cc
+
+    Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// This program sets up the initial pseudofermion field |Phi> = Meofa^{-1/2}*|eta>, and
+// then uses this Phi to compute the action <Phi|Meofa|Phi>.
+// If all is working, one should find that <eta|eta> = <Phi|Meofa|Phi>.
+//////////////////////////////////////////////////////////////////////////////////////////
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+// Parameters for test
+const std::vector<int> grid_dim = { 8, 8, 8, 8 };
+const int              Ls       = 8;
+const int              Npoles   = 12;
+const RealD            mf       = 0.01;
+const RealD            mpv      = 1.0;
+const RealD            M5       = 1.8;
+
+int main(int argc, char** argv)
+{
+  Grid_init(&argc, &argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is set up to use " << threads << " threads" << std::endl;
+
+  // Initialize spacetime grid
+  std::cout << GridLogMessage << "Lattice dimensions: " << grid_dim << "  Ls: " << Ls << std::endl;
+  GridCartesian*         UGrid   = SpaceTimeGrid::makeFourDimGrid(grid_dim,
+                                      GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian*         FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Set up RNGs
+  std::vector<int> seeds4({1, 2, 3, 4});
+  std::vector<int> seeds5({5, 6, 7, 8});
+  GridParallelRNG RNG5(FGrid);
+  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);
+  RNG4.SeedFixedIntegers(seeds4);
+
+  // Random gauge field
+  LatticeGaugeField Umu(UGrid);
+  SU3::HotConfiguration(RNG4, Umu);
+
+  DomainWallEOFAFermionR Lop(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf,  mf, mpv,  0.0, -1, M5);
+  DomainWallEOFAFermionR Rop(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mpv, mf, mpv, -1.0,  1, M5);
+
+  // Construct the action and test the heatbath (zero initial guess)
+  {
+    OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, Npoles);
+    ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
+    ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, false);
+
+    Meofa.refresh(Umu, RNG5);
+    printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
+  }
+
+  // Construct the action and test the heatbath (forecasted initial guesses)
+  {
+    OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, Npoles);
+    ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
+    ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, true);
+
+    Meofa.refresh(Umu, RNG5);
+    printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
+  }
+
+  return 0;
+}
diff --git a/tests/debug/Test_heatbath_dwf_eofa_gparity.cc b/tests/debug/Test_heatbath_dwf_eofa_gparity.cc
new file mode 100644
index 00000000..5c9d4923
--- /dev/null
+++ b/tests/debug/Test_heatbath_dwf_eofa_gparity.cc
@@ -0,0 +1,108 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid
+
+    Source file: ./tests/debug/Test_heatbath_dwf_eofa.cc
+
+    Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// This program sets up the initial pseudofermion field |Phi> = Meofa^{-1/2}*|eta>, and
+// then uses this Phi to compute the action <Phi|Meofa|Phi>.
+// If all is working, one should find that <eta|eta> = <Phi|Meofa|Phi>.
+//////////////////////////////////////////////////////////////////////////////////////////
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+typedef GparityWilsonImplR FermionImplPolicy;
+typedef GparityDomainWallEOFAFermionR FermionAction;
+typedef typename FermionAction::FermionField FermionField;
+
+// Parameters for test
+const std::vector<int> grid_dim = { 8, 8, 8, 8 };
+const int              Ls       = 8;
+const int              Npoles   = 12;
+const RealD            mf       = 0.01;
+const RealD            mpv      = 1.0;
+const RealD            M5       = 1.8;
+
+int main(int argc, char** argv)
+{
+  Grid_init(&argc, &argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is set up to use " << threads << " threads" << std::endl;
+
+  // Initialize spacetime grid
+  std::cout << GridLogMessage << "Lattice dimensions: " << grid_dim << "  Ls: " << Ls << std::endl;
+  GridCartesian*         UGrid   = SpaceTimeGrid::makeFourDimGrid(grid_dim,
+                                      GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian*         FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Set up RNGs
+  std::vector<int> seeds4({1, 2, 3, 4});
+  std::vector<int> seeds5({5, 6, 7, 8});
+  GridParallelRNG RNG5(FGrid);
+  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);
+  RNG4.SeedFixedIntegers(seeds4);
+
+  // Random gauge field
+  LatticeGaugeField Umu(UGrid);
+  SU3::HotConfiguration(RNG4, Umu);
+
+  // GparityDomainWallFermionR::ImplParams params;
+  FermionAction::ImplParams params;
+  FermionAction Lop(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf,  mf, mpv,  0.0, -1, M5, params);
+  FermionAction Rop(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mpv, mf, mpv, -1.0,  1, M5, params);
+
+  // Construct the action and test the heatbath (zero initial guess)
+  {
+    OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, Npoles);
+    ConjugateGradient<FermionField> CG(1.0e-12, 5000);
+    ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, false);
+
+    Meofa.refresh(Umu, RNG5);
+    printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
+  }
+
+  // Construct the action and test the heatbath (forecasted initial guesses)
+  {
+    OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, Npoles);
+    ConjugateGradient<FermionField> CG(1.0e-12, 5000);
+    ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, true);
+
+    Meofa.refresh(Umu, RNG5);
+    printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
+  }
+
+  return 0;
+}

From 47a12ec7b560c60d3dde74392598517c98d8b83d Mon Sep 17 00:00:00 2001
From: David Murphy <djm2131@columbia.edu>
Date: Wed, 16 Aug 2017 19:50:08 -0400
Subject: [PATCH 038/115] Implement EOFA pseudofermion force and Shamir tests
 for G-parity and non G-parity cases

---
 .../pseudofermion/ExactOneFlavourRatio.h      |  36 +++-
 tests/forces/Test_dwf_force_eofa.cc           | 164 +++++++++++++++++
 tests/forces/Test_dwf_gpforce_eofa.cc         | 169 ++++++++++++++++++
 3 files changed, 368 insertions(+), 1 deletion(-)
 create mode 100644 tests/forces/Test_dwf_force_eofa.cc
 create mode 100644 tests/forces/Test_dwf_gpforce_eofa.cc

diff --git a/lib/qcd/action/pseudofermion/ExactOneFlavourRatio.h b/lib/qcd/action/pseudofermion/ExactOneFlavourRatio.h
index 38c7380c..9c1e2921 100644
--- a/lib/qcd/action/pseudofermion/ExactOneFlavourRatio.h
+++ b/lib/qcd/action/pseudofermion/ExactOneFlavourRatio.h
@@ -118,7 +118,7 @@ namespace QCD{
         RealD scale = std::sqrt(0.5);
         gaussian(pRNG,eta);
         eta = eta * scale;
-        printf("Heatbath source vector: <\eta|\eta> = %1.15e\n", norm2(eta));
+        printf("Heatbath source vector: <\\eta|\\eta> = %1.15e\n", norm2(eta));
 
         // \Phi = ( \alpha_{0} + \sum_{k=1}^{N_{p}} \alpha_{l} * \gamma_{l} ) * \eta
         RealD N(PowerNegHalf.norm);
@@ -223,6 +223,40 @@ namespace QCD{
       // EOFA pseudofermion force: see Eqns. (34)-(36) of arXiv:1706.05843
       virtual void deriv(const GaugeField& U, GaugeField& dSdU)
       {
+        Lop.ImportGauge(U);
+        Rop.ImportGauge(U);
+
+        FermionField spProj_Phi      (Lop.FermionGrid());
+        FermionField Omega_spProj_Phi(Lop.FermionGrid());
+        FermionField CG_src          (Lop.FermionGrid());
+        FermionField Chi             (Lop.FermionGrid());
+        FermionField g5_R5_Chi       (Lop.FermionGrid());
+
+        GaugeField force(Lop.GaugeGrid());
+
+        // LH: dSdU = k \chi_{L}^{\dagger} \gamma_{5} R_{5} ( \partial_{x,\mu} D_{w} ) \chi_{L}
+        //     \chi_{L} = H(mf)^{-1} \Omega_{-} P_{-} \Phi
+        spProj(Phi, spProj_Phi, -1, Lop.Ls);
+        Lop.Omega(spProj_Phi, Omega_spProj_Phi, -1, 0);
+        G5R5(CG_src, Omega_spProj_Phi);
+        spProj_Phi = zero;
+        Solver(Lop, CG_src, spProj_Phi);
+        Lop.Dtilde(spProj_Phi, Chi);
+        G5R5(g5_R5_Chi, Chi);
+        Lop.MDeriv(force, g5_R5_Chi, Chi, DaggerNo);
+        dSdU = Lop.k * force;
+
+        // RH: dSdU = dSdU - k \chi_{R}^{\dagger} \gamma_{5} R_{5} ( \partial_{x,\mu} D_{w} ) \chi_{}
+        //     \chi_{R} = ( H(mb) - \Delta_{+}(mf,mb) P_{+} )^{-1} \Omega_{+} P_{+} \Phi
+        spProj(Phi, spProj_Phi, 1, Rop.Ls);
+        Rop.Omega(spProj_Phi, Omega_spProj_Phi, 1, 0);
+        G5R5(CG_src, Omega_spProj_Phi);
+        spProj_Phi = zero;
+        Solver(Rop, CG_src, spProj_Phi);
+        Rop.Dtilde(spProj_Phi, Chi);
+        G5R5(g5_R5_Chi, Chi);
+        Lop.MDeriv(force, g5_R5_Chi, Chi, DaggerNo);
+        dSdU = dSdU - Rop.k * force;
       };
   };
 }}
diff --git a/tests/forces/Test_dwf_force_eofa.cc b/tests/forces/Test_dwf_force_eofa.cc
new file mode 100644
index 00000000..f17579ae
--- /dev/null
+++ b/tests/forces/Test_dwf_force_eofa.cc
@@ -0,0 +1,164 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/forces/Test_dwf_force_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+int main (int argc, char** argv)
+{
+  Grid_init(&argc, &argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+
+  const int Ls = 8;
+
+  GridCartesian         *UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         *FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Want a different conf at every run
+  // First create an instance of an engine.
+  std::random_device rnd_device;
+  // Specify the engine and distribution.
+  std::mt19937 mersenne_engine(rnd_device());
+  std::uniform_int_distribution<int> dist(1, 100);
+
+  auto gen = std::bind(dist, mersenne_engine);
+  std::vector<int> seeds4(4);
+  generate(begin(seeds4), end(seeds4), gen);
+
+  //std::vector<int> seeds4({1,2,3,5});
+  std::vector<int> seeds5({5,6,7,8});
+  GridParallelRNG RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+  LatticeFermion phi        (FGrid);  gaussian(RNG5, phi);
+  LatticeFermion Mphi       (FGrid);
+  LatticeFermion MphiPrime  (FGrid);
+
+  LatticeGaugeField U(UGrid);
+  SU3::HotConfiguration(RNG4,U);
+
+  ////////////////////////////////////
+  // Unmodified matrix element
+  ////////////////////////////////////
+  RealD mf = 0.01;
+  RealD mb = 1.0;
+  RealD M5 = 1.8;
+  DomainWallEOFAFermionR Lop(U, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, mf, mb, 0.0, -1, M5);
+  DomainWallEOFAFermionR Rop(U, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, mf, mb, -1.0, 1, M5);
+  OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, 12);
+  ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
+  ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, true);
+
+  Meofa.refresh(U, RNG5);
+  RealD S = Meofa.S(U); // pdag M p
+
+  // get the deriv of phidag M phi with respect to "U"
+  LatticeGaugeField UdSdU(UGrid);
+  Meofa.deriv(U, UdSdU);
+
+  ////////////////////////////////////
+  // Modify the gauge field a little
+  ////////////////////////////////////
+  RealD dt = 0.0001;
+
+  LatticeColourMatrix mommu(UGrid);
+  LatticeColourMatrix forcemu(UGrid);
+  LatticeGaugeField mom(UGrid);
+  LatticeGaugeField Uprime(UGrid);
+
+  for(int mu=0; mu<Nd; mu++){
+
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+
+    PokeIndex<LorentzIndex>(mom, mommu, mu);
+
+    // fourth order exponential approx
+    parallel_for(auto i=mom.begin(); i<mom.end(); i++){
+      Uprime[i](mu) = U[i](mu) + mom[i](mu)*U[i](mu)*dt + mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt/2.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt/6.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt/24.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt/120.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt*dt/720.0);
+    }
+  }
+
+  /*Ddwf.ImportGauge(Uprime);
+  Ddwf.M          (phi,MphiPrime);
+
+  ComplexD Sprime    = innerProduct(MphiPrime   ,MphiPrime);*/
+  RealD Sprime = Meofa.S(Uprime);
+
+  //////////////////////////////////////////////
+  // Use derivative to estimate dS
+  //////////////////////////////////////////////
+
+  LatticeComplex dS(UGrid);
+  dS = zero;
+  for(int mu=0; mu<Nd; mu++){
+    mommu = PeekIndex<LorentzIndex>(UdSdU, mu);
+    mommu = Ta(mommu)*2.0;
+    PokeIndex<LorentzIndex>(UdSdU, mommu, mu);
+  }
+
+  for(int mu=0; mu<Nd; mu++){
+    forcemu = PeekIndex<LorentzIndex>(UdSdU, mu);
+    mommu   = PeekIndex<LorentzIndex>(mom, mu);
+
+    // Update PF action density
+    dS = dS + trace(mommu*forcemu)*dt;
+  }
+
+  ComplexD dSpred = sum(dS);
+
+  /*std::cout << GridLogMessage << " S      " << S << std::endl;
+  std::cout << GridLogMessage << " Sprime " << Sprime << std::endl;
+  std::cout << GridLogMessage << "dS      " << Sprime-S << std::endl;
+  std::cout << GridLogMessage << "predict dS    " << dSpred << std::endl;*/
+  printf("\nS = %1.15e\n", S);
+  printf("Sprime = %1.15e\n", Sprime);
+  printf("dS = %1.15e\n", Sprime - S);
+  printf("real(dS_predict) = %1.15e\n", dSpred.real());
+  printf("imag(dS_predict) = %1.15e\n\n", dSpred.imag());
+
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
+
+  std::cout << GridLogMessage << "Done" << std::endl;
+  Grid_finalize();
+}
diff --git a/tests/forces/Test_dwf_gpforce_eofa.cc b/tests/forces/Test_dwf_gpforce_eofa.cc
new file mode 100644
index 00000000..50789c89
--- /dev/null
+++ b/tests/forces/Test_dwf_gpforce_eofa.cc
@@ -0,0 +1,169 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/forces/Test_dwf_force_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+typedef GparityWilsonImplR FermionImplPolicy;
+typedef GparityDomainWallEOFAFermionR FermionAction;
+typedef typename FermionAction::FermionField FermionField;
+
+int main (int argc, char** argv)
+{
+  Grid_init(&argc, &argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+
+  const int Ls = 8;
+
+  GridCartesian         *UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         *FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Want a different conf at every run
+  // First create an instance of an engine.
+  std::random_device rnd_device;
+  // Specify the engine and distribution.
+  std::mt19937 mersenne_engine(rnd_device());
+  std::uniform_int_distribution<int> dist(1, 100);
+
+  auto gen = std::bind(dist, mersenne_engine);
+  std::vector<int> seeds4(4);
+  generate(begin(seeds4), end(seeds4), gen);
+
+  //std::vector<int> seeds4({1,2,3,5});
+  std::vector<int> seeds5({5,6,7,8});
+  GridParallelRNG RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+  LatticeFermion phi        (FGrid);  gaussian(RNG5, phi);
+  LatticeFermion Mphi       (FGrid);
+  LatticeFermion MphiPrime  (FGrid);
+
+  LatticeGaugeField U(UGrid);
+  SU3::HotConfiguration(RNG4,U);
+
+  ////////////////////////////////////
+  // Unmodified matrix element
+  ////////////////////////////////////
+  RealD mf = 0.01;
+  RealD mb = 1.0;
+  RealD M5 = 1.8;
+  FermionAction::ImplParams params;
+  FermionAction Lop(U, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, mf, mb, 0.0, -1, M5, params);
+  FermionAction Rop(U, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, mf, mb, -1.0, 1, M5, params);
+  OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, 12);
+  ConjugateGradient<FermionField> CG(1.0e-12, 5000);
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, true);
+
+  Meofa.refresh(U, RNG5);
+  RealD S = Meofa.S(U); // pdag M p
+
+  // get the deriv of phidag M phi with respect to "U"
+  LatticeGaugeField UdSdU(UGrid);
+  Meofa.deriv(U, UdSdU);
+
+  ////////////////////////////////////
+  // Modify the gauge field a little
+  ////////////////////////////////////
+  RealD dt = 0.0001;
+
+  LatticeColourMatrix mommu(UGrid);
+  LatticeColourMatrix forcemu(UGrid);
+  LatticeGaugeField mom(UGrid);
+  LatticeGaugeField Uprime(UGrid);
+
+  for(int mu=0; mu<Nd; mu++){
+
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+
+    PokeIndex<LorentzIndex>(mom, mommu, mu);
+
+    // fourth order exponential approx
+    parallel_for(auto i=mom.begin(); i<mom.end(); i++){
+      Uprime[i](mu) = U[i](mu) + mom[i](mu)*U[i](mu)*dt + mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt/2.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt/6.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt/24.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt/120.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt*dt/720.0);
+    }
+  }
+
+  /*Ddwf.ImportGauge(Uprime);
+  Ddwf.M          (phi,MphiPrime);
+
+  ComplexD Sprime    = innerProduct(MphiPrime   ,MphiPrime);*/
+  RealD Sprime = Meofa.S(Uprime);
+
+  //////////////////////////////////////////////
+  // Use derivative to estimate dS
+  //////////////////////////////////////////////
+
+  LatticeComplex dS(UGrid);
+  dS = zero;
+  for(int mu=0; mu<Nd; mu++){
+    mommu = PeekIndex<LorentzIndex>(UdSdU, mu);
+    mommu = Ta(mommu)*2.0;
+    PokeIndex<LorentzIndex>(UdSdU, mommu, mu);
+  }
+
+  for(int mu=0; mu<Nd; mu++){
+    forcemu = PeekIndex<LorentzIndex>(UdSdU, mu);
+    mommu   = PeekIndex<LorentzIndex>(mom, mu);
+
+    // Update PF action density
+    dS = dS + trace(mommu*forcemu)*dt;
+  }
+
+  ComplexD dSpred = sum(dS);
+
+  /*std::cout << GridLogMessage << " S      " << S << std::endl;
+  std::cout << GridLogMessage << " Sprime " << Sprime << std::endl;
+  std::cout << GridLogMessage << "dS      " << Sprime-S << std::endl;
+  std::cout << GridLogMessage << "predict dS    " << dSpred << std::endl;*/
+  printf("\nS = %1.15e\n", S);
+  printf("Sprime = %1.15e\n", Sprime);
+  printf("dS = %1.15e\n", Sprime - S);
+  printf("real(dS_predict) = %1.15e\n", dSpred.real());
+  printf("imag(dS_predict) = %1.15e\n\n", dSpred.imag());
+
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
+
+  std::cout << GridLogMessage << "Done" << std::endl;
+  Grid_finalize();
+}

From d9d3d30cc7334bdd7c046f6073b43e339dbabd21 Mon Sep 17 00:00:00 2001
From: David Murphy <djm2131@columbia.edu>
Date: Wed, 16 Aug 2017 20:57:51 -0400
Subject: [PATCH 039/115] Minor clean-up

---
 lib/qcd/action/fermion/DomainWallEOFAFermion.h | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermion.h b/lib/qcd/action/fermion/DomainWallEOFAFermion.h
index d48e3b8f..5362cda8 100644
--- a/lib/qcd/action/fermion/DomainWallEOFAFermion.h
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermion.h
@@ -35,24 +35,6 @@ See the full license in the file "LICENSE" in the top level distribution directo
 namespace Grid {
 namespace QCD {
 
-  /*template<typename T> struct switcheroo {
-    static inline int iscomplex() { return 0; }
-    template<class vec>
-    static inline vec mult(vec a, vec b){ return real_mult(a,b); }
-  };
-
-  template<> struct switcheroo<ComplexD> {
-    static inline int iscomplex() { return 1; }
-    template<class vec>
-    static inline vec mult(vec a, vec b){ return a*b; }
-  };
-
-  template<> struct switcheroo<ComplexF> {
-    static inline int iscomplex() { return 1; }
-    template<class vec>
-    static inline vec mult(vec a, vec b) { return a*b; }
-  };*/
-
   template<class Impl>
   class DomainWallEOFAFermion : public AbstractEOFAFermion<Impl>
   {

From e140b3f802549b63809fcf0ad88657f19497a75c Mon Sep 17 00:00:00 2001
From: David Murphy <djm2131@columbia.edu>
Date: Wed, 16 Aug 2017 23:36:23 -0400
Subject: [PATCH 040/115] Beginning to re-import Mobius EOFA

---
 lib/qcd/action/fermion/Fermion.h            |  31 +-
 lib/qcd/action/fermion/MobiusEOFAFermion.cc | 469 ++++++++++++++++++++
 lib/qcd/action/fermion/MobiusEOFAFermion.h  | 133 ++++++
 3 files changed, 631 insertions(+), 2 deletions(-)
 create mode 100644 lib/qcd/action/fermion/MobiusEOFAFermion.cc
 create mode 100644 lib/qcd/action/fermion/MobiusEOFAFermion.h

diff --git a/lib/qcd/action/fermion/Fermion.h b/lib/qcd/action/fermion/Fermion.h
index ac2a94b2..99d97895 100644
--- a/lib/qcd/action/fermion/Fermion.h
+++ b/lib/qcd/action/fermion/Fermion.h
@@ -1,6 +1,6 @@
 /*************************************************************************************
 
-    Grid physics library, www.github.com/paboyle/Grid 
+    Grid physics library, www.github.com/paboyle/Grid
 
     Source file: ./lib/qcd/action/fermion/Fermion_base_aggregate.h
 
@@ -38,6 +38,8 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
 // - ContinuedFractionFermion5D.cc
 // - WilsonFermion.cc
 // - WilsonKernels.cc
+// - DomainWallEOFAFermion.cc
+// - MobiusEOFAFermion.cc
 //
 // The explicit instantiation is only avoidable if we move this source to headers and end up with include/parse/recompile
 // for EVERY .cc file. This define centralises the list and restores global push of impl cases
@@ -57,6 +59,7 @@ Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
 #include <Grid/qcd/action/fermion/DomainWallFermion.h>
 #include <Grid/qcd/action/fermion/DomainWallEOFAFermion.h>
 #include <Grid/qcd/action/fermion/MobiusFermion.h>
+#include <Grid/qcd/action/fermion/MobiusEOFAFermion.h>
 #include <Grid/qcd/action/fermion/ZMobiusFermion.h>
 #include <Grid/qcd/action/fermion/SchurDiagTwoKappa.h>
 #include <Grid/qcd/action/fermion/ScaledShamirFermion.h>
@@ -129,6 +132,14 @@ typedef MobiusFermion<WilsonImplRL> MobiusFermionRL;
 typedef MobiusFermion<WilsonImplFH> MobiusFermionFH;
 typedef MobiusFermion<WilsonImplDF> MobiusFermionDF;
 
+typedef MobiusEOFAFermion<WilsonImplR> MobiusEOFAFermionR;
+typedef MobiusEOFAFermion<WilsonImplF> MobiusEOFAFermionF;
+typedef MobiusEOFAFermion<WilsonImplD> MobiusEOFAFermionD;
+
+typedef MobiusEOFAFermion<WilsonImplRL> MobiusEOFAFermionRL;
+typedef MobiusEOFAFermion<WilsonImplFH> MobiusEOFAFermionFH;
+typedef MobiusEOFAFermion<WilsonImplDF> MobiusEOFAFermionDF;
+
 typedef ZMobiusFermion<ZWilsonImplR> ZMobiusFermionR;
 typedef ZMobiusFermion<ZWilsonImplF> ZMobiusFermionF;
 typedef ZMobiusFermion<ZWilsonImplD> ZMobiusFermionD;
@@ -137,7 +148,7 @@ typedef ZMobiusFermion<ZWilsonImplRL> ZMobiusFermionRL;
 typedef ZMobiusFermion<ZWilsonImplFH> ZMobiusFermionFH;
 typedef ZMobiusFermion<ZWilsonImplDF> ZMobiusFermionDF;
 
-// Ls vectorised 
+// Ls vectorised
 typedef DomainWallFermion<DomainWallVec5dImplR> DomainWallFermionVec5dR;
 typedef DomainWallFermion<DomainWallVec5dImplF> DomainWallFermionVec5dF;
 typedef DomainWallFermion<DomainWallVec5dImplD> DomainWallFermionVec5dD;
@@ -162,6 +173,14 @@ typedef MobiusFermion<DomainWallVec5dImplRL> MobiusFermionVec5dRL;
 typedef MobiusFermion<DomainWallVec5dImplFH> MobiusFermionVec5dFH;
 typedef MobiusFermion<DomainWallVec5dImplDF> MobiusFermionVec5dDF;
 
+typedef MobiusEOFAFermion<DomainWallVec5dImplR> MobiusEOFAFermionVec5dR;
+typedef MobiusEOFAFermion<DomainWallVec5dImplF> MobiusEOFAFermionVec5dF;
+typedef MobiusEOFAFermion<DomainWallVec5dImplD> MobiusEOFAFermionVec5dD;
+
+typedef MobiusEOFAFermion<DomainWallVec5dImplRL> MobiusEOFAFermionVec5dRL;
+typedef MobiusEOFAFermion<DomainWallVec5dImplFH> MobiusEOFAFermionVec5dFH;
+typedef MobiusEOFAFermion<DomainWallVec5dImplDF> MobiusEOFAFermionVec5dDF;
+
 typedef ZMobiusFermion<ZDomainWallVec5dImplR> ZMobiusFermionVec5dR;
 typedef ZMobiusFermion<ZDomainWallVec5dImplF> ZMobiusFermionVec5dF;
 typedef ZMobiusFermion<ZDomainWallVec5dImplD> ZMobiusFermionVec5dD;
@@ -246,6 +265,14 @@ typedef MobiusFermion<GparityWilsonImplRL> GparityMobiusFermionRL;
 typedef MobiusFermion<GparityWilsonImplFH> GparityMobiusFermionFH;
 typedef MobiusFermion<GparityWilsonImplDF> GparityMobiusFermionDF;
 
+typedef MobiusEOFAFermion<GparityWilsonImplR> GparityMobiusFermionR;
+typedef MobiusEOFAFermion<GparityWilsonImplF> GparityMobiusFermionF;
+typedef MobiusEOFAFermion<GparityWilsonImplD> GparityMobiusFermionD;
+
+typedef MobiusEOFAFermion<GparityWilsonImplRL> GparityMobiusEOFAFermionRL;
+typedef MobiusEOFAFermion<GparityWilsonImplFH> GparityMobiusEOFAFermionFH;
+typedef MobiusEOFAFermion<GparityWilsonImplDF> GparityMobiusEOFAFermionDF;
+
 typedef ImprovedStaggeredFermion<StaggeredImplR> ImprovedStaggeredFermionR;
 typedef ImprovedStaggeredFermion<StaggeredImplF> ImprovedStaggeredFermionF;
 typedef ImprovedStaggeredFermion<StaggeredImplD> ImprovedStaggeredFermionD;
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermion.cc b/lib/qcd/action/fermion/MobiusEOFAFermion.cc
new file mode 100644
index 00000000..9d0d8b11
--- /dev/null
+++ b/lib/qcd/action/fermion/MobiusEOFAFermion.cc
@@ -0,0 +1,469 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/MobiusEOFAFermion.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid_Eigen_Dense.h>
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/MobiusEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+  template<class Impl>
+    MobiusEOFAFermion<Impl>::MobiusEOFAFermion(
+      GaugeField            &_Umu,
+      GridCartesian         &FiveDimGrid,
+      GridRedBlackCartesian &FiveDimRedBlackGrid,
+      GridCartesian         &FourDimGrid,
+      GridRedBlackCartesian &FourDimRedBlackGrid,
+      RealD _mq1, RealD _mq2, RealD _mq3,
+      RealD _shift, int _pm, RealD _M5,
+      RealD _b, RealD _c, const ImplParams &p) :
+    AbstractEOFAFermion<Impl>(_Umu, FiveDimGrid, FiveDimRedBlackGrid,
+        FourDimGrid, FourDimRedBlackGrid, _mq1, _mq2, _mq3,
+        _shift, _pm, _M5, _b, _c, p)
+    {
+      int Ls = this->Ls;
+
+      RealD eps = 1.0;
+      Approx::zolotarev_data *zdata = Approx::higham(eps, this->Ls);
+      assert(zdata->n == this->Ls);
+
+      std::cout << GridLogMessage << "MobiusEOFAFermion (b=" << _b <<
+        ",c=" << _c << ") with Ls=" << Ls << std::endl;
+      this->SetCoefficientsTanh(zdata, _b, _c);
+      std::cout << GridLogMessage << "EOFA parameters: (mq1=" << _mq1 <<
+        ",mq2=" << _mq2 << ",mq3=" << _mq3 << ",shift=" << _shift <<
+        ",pm=" << _pm << ")" << std::endl;
+
+      Approx::zolotarev_free(zdata);
+
+      if(_shift != 0.0){
+        SetCoefficientsPrecondShiftOps();
+      } else {
+        Mooee_shift.resize(Ls, 0.0);
+        MooeeInv_shift_lc.resize(Ls, 0.0);
+        MooeeInv_shift_norm.resize(Ls, 0.0);
+        MooeeInvDag_shift_lc.resize(Ls, 0.0);
+        MooeeInvDag_shift_norm.resize(Ls, 0.0);
+      }
+    }
+
+    /***************************************************************
+    /* Additional EOFA operators only called outside the inverter.
+    /* Since speed is not essential, simple axpby-style
+    /* implementations should be fine.
+    /***************************************************************/
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::Omega(const FermionField& psi, FermionField& Din, int sign, int dag)
+    {
+      int Ls = this->Ls;
+      RealD alpha = this->alpha;
+
+      Din = zero;
+      if((sign == 1) && (dag == 0)) { // \Omega_{+}
+        for(int s=0; s<Ls; ++s){
+          axpby_ssp(Din, 0.0, psi, 2.0*std::pow(1.0-alpha,Ls-s-1)/std::pow(1.0+alpha,Ls-s), psi, s, 0);
+        }
+      } else if((sign == -1) && (dag == 0)) { // \Omega_{-}
+        for(int s=0; s<Ls; ++s){
+          axpby_ssp(Din, 0.0, psi, 2.0*std::pow(1.0-alpha,s)/std::pow(1.0+alpha,s+1), psi, s, 0);
+        }
+      } else if((sign == 1 ) && (dag == 1)) { // \Omega_{+}^{\dagger}
+        for(int sp=0; sp<Ls; ++sp){
+          axpby_ssp(Din, 1.0, Din, 2.0*std::pow(1.0-alpha,Ls-sp-1)/std::pow(1.0+alpha,Ls-sp), psi, 0, sp);
+        }
+      } else if((sign == -1) && (dag == 1)) { // \Omega_{-}^{\dagger}
+        for(int sp=0; sp<Ls; ++sp){
+          axpby_ssp(Din, 1.0, Din, 2.0*std::pow(1.0-alpha,sp)/std::pow(1.0+alpha,sp+1), psi, 0, sp);
+        }
+      }
+    }
+
+    // This is the operator relating the usual Ddwf to TWQCD's EOFA Dirac operator (arXiv:1706.05843, Eqn. 6).
+    // It also relates the preconditioned and unpreconditioned systems described in Appendix B.2.
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::Dtilde(const FermionField& psi, FermionField& chi)
+    {
+      int Ls    = this->Ls;
+      RealD b   = 0.5 * ( 1.0 + this->alpha );
+      RealD c   = 0.5 * ( 1.0 - this->alpha );
+      RealD mq1 = this->mq1;
+
+      for(int s=0; s<Ls; ++s){
+        if(s == 0) {
+          axpby_ssp_pminus(chi, b, psi, -c, psi, s, s+1);
+          axpby_ssp_pplus (chi, 1.0, chi, mq1*c, psi, s, Ls-1);
+        } else if(s == (Ls-1)) {
+          axpby_ssp_pminus(chi, b, psi, mq1*c, psi, s, 0);
+          axpby_ssp_pplus (chi, 1.0, chi, -c, psi, s, s-1);
+        } else {
+          axpby_ssp_pminus(chi, b, psi, -c, psi, s, s+1);
+          axpby_ssp_pplus (chi, 1.0, chi, -c, psi, s, s-1);
+        }
+      }
+    }
+
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::DtildeInv(const FermionField& psi, FermionField& chi){ }
+
+    /*****************************************************************************************************/
+
+    template<class Impl>
+    RealD MobiusEOFAFermion<Impl>::M(const FermionField& psi, FermionField& chi)
+    {
+      int Ls = this->Ls;
+
+      FermionField Din(psi._grid);
+
+      this->Meooe5D(psi, Din);
+      this->DW(Din, chi, DaggerNo);
+      axpby(chi, 1.0, 1.0, chi, psi);
+      this->M5D(psi, chi);
+      return(norm2(chi));
+    }
+
+    template<class Impl>
+    RealD MobiusEOFAFermion<Impl>::Mdag(const FermionField& psi, FermionField& chi)
+    {
+      int Ls = this->Ls;
+
+      FermionField Din(psi._grid);
+
+      this->DW(psi, Din, DaggerYes);
+      this->MeooeDag5D(Din, chi);
+      this->M5Ddag(psi, chi);
+      axpby(chi, 1.0, 1.0, chi, psi);
+      return(norm2(chi));
+    }
+
+    /********************************************************************
+    /* Performance critical fermion operators called inside the inverter
+    /********************************************************************/
+
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::M5D(const FermionField& psi, FermionField& chi)
+    {
+        int Ls = this->Ls;
+
+        std::vector<Coeff_t> diag(Ls,1.0);
+        std::vector<Coeff_t> upper(Ls,-1.0);  upper[Ls-1] = this->mq1;
+        std::vector<Coeff_t> lower(Ls,-1.0);  lower[0]    = this->mq1;
+
+        // no shift term
+        if(this->shift == 0.0){ this->M5D(psi, chi, chi, lower, diag, upper); }
+
+        // fused M + shift operation
+        else{ this->M5D_shift(psi, chi, chi, lower, diag, upper, Mooee_shift); }
+    }
+
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::M5Ddag(const FermionField& psi, FermionField& chi)
+    {
+        int   Ls    = this->Ls;
+        int   pm    = this->pm;
+        RealD shift = this->shift;
+        RealD mq1   = this->mq1;
+        RealD mq2   = this->mq2;
+        RealD mq3   = this->mq3;
+
+        // coefficients for shift operator ( = shift*\gamma_{5}*R_{5}*\Delta_{\pm}(mq2,mq3)*P_{\pm} )
+        Coeff_t shiftp(0.0), shiftm(0.0);
+        if(shift != 0.0){
+          if(pm == 1){ shiftp = shift*(mq3-mq2); }
+          else{ shiftm = -shift*(mq3-mq2); }
+        }
+
+        std::vector<Coeff_t> diag(Ls,1.0);
+        std::vector<Coeff_t> upper(Ls,-1.0); upper[Ls-1] = mq1 + shiftp;
+        std::vector<Coeff_t> lower(Ls,-1.0); lower[0]    = mq1 + shiftm;
+
+        #if(0)
+            std::cout << GridLogMessage << "MobiusEOFAFermion::M5Ddag(FF&,FF&):" << std::endl;
+            for(int i=0; i<diag.size(); ++i){
+                std::cout << GridLogMessage << "diag[" << i << "] =" << diag[i] << std::endl;
+            }
+            for(int i=0; i<upper.size(); ++i){
+                std::cout << GridLogMessage << "upper[" << i << "] =" << upper[i] << std::endl;
+            }
+            for(int i=0; i<lower.size(); ++i){
+                std::cout << GridLogMessage << "lower[" << i << "] =" << lower[i] << std::endl;
+            }
+        #endif
+
+        this->M5Ddag(psi, chi, chi, lower, diag, upper);
+    }
+
+    // half checkerboard operations
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::Mooee(const FermionField& psi, FermionField& chi)
+    {
+        int Ls = this->Ls;
+
+        std::vector<Coeff_t> diag = this->bee;
+        std::vector<Coeff_t> upper(Ls);
+        std::vector<Coeff_t> lower(Ls);
+
+        for(int s=0; s<Ls; s++){
+          upper[s] = -this->cee[s];
+          lower[s] = -this->cee[s];
+        }
+        upper[Ls-1] = this->dm;
+        lower[0]    = this->dp;
+
+        this->M5D(psi, psi, chi, lower, diag, upper);
+    }
+
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::MooeeDag(const FermionField& psi, FermionField& chi)
+    {
+        int Ls = this->Ls;
+
+        std::vector<Coeff_t> diag = this->bee;
+        std::vector<Coeff_t> upper(Ls);
+        std::vector<Coeff_t> lower(Ls);
+
+        for(int s=0; s<Ls; s++){
+          upper[s] = -this->cee[s];
+          lower[s] = -this->cee[s];
+        }
+        upper[Ls-1] = this->dp;
+        lower[0]    = this->dm;
+
+        this->M5Ddag(psi, psi, chi, lower, diag, upper);
+    }
+
+    /****************************************************************************************/
+
+    //Zolo
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::SetCoefficientsInternal(RealD zolo_hi, std::vector<Coeff_t>& gamma, RealD b, RealD c)
+    {
+        int   Ls    = this->Ls;
+        int   pm    = this->pm;
+        RealD mq1   = this->mq1;
+        RealD mq2   = this->mq2;
+        RealD mq3   = this->mq3;
+        RealD shift = this->shift;
+
+        ////////////////////////////////////////////////////////
+        // Constants for the preconditioned matrix Cayley form
+        ////////////////////////////////////////////////////////
+        this->bs.resize(Ls);
+        this->cs.resize(Ls);
+        this->aee.resize(Ls);
+        this->aeo.resize(Ls);
+        this->bee.resize(Ls);
+        this->beo.resize(Ls);
+        this->cee.resize(Ls);
+        this->ceo.resize(Ls);
+
+        for(int i=0; i<Ls; ++i){
+          this->bee[i] = 4.0 - this->M5 + 1.0;
+          this->cee[i] = 1.0;
+        }
+
+        for(int i=0; i<Ls; ++i){
+          this->aee[i] = this->cee[i];
+          this->bs[i] = this->beo[i] = 1.0;
+          this->cs[i] = this->ceo[i] = 0.0;
+        }
+
+        //////////////////////////////////////////
+        // EOFA shift terms
+        //////////////////////////////////////////
+        if(pm == 1){
+          this->dp = mq1*this->cee[0] + shift*(mq3-mq2);
+          this->dm = mq1*this->cee[Ls-1];
+        } else if(this->pm == -1) {
+          this->dp = mq1*this->cee[0];
+          this->dm = mq1*this->cee[Ls-1] - shift*(mq3-mq2);
+        } else {
+          this->dp = mq1*this->cee[0];
+          this->dm = mq1*this->cee[Ls-1];
+        }
+
+        //////////////////////////////////////////
+        // LDU decomposition of eeoo
+        //////////////////////////////////////////
+        this->dee.resize(Ls+1);
+        this->lee.resize(Ls);
+        this->leem.resize(Ls);
+        this->uee.resize(Ls);
+        this->ueem.resize(Ls);
+
+        for(int i=0; i<Ls; ++i){
+
+          if(i < Ls-1){
+
+            this->lee[i] = -this->cee[i+1]/this->bee[i]; // sub-diag entry on the ith column
+
+            this->leem[i] = this->dm/this->bee[i];
+            for(int j=0; j<i; j++){ this->leem[i] *= this->aee[j]/this->bee[j]; }
+
+            this->dee[i] = this->bee[i];
+
+            this->uee[i] = -this->aee[i]/this->bee[i];   // up-diag entry on the ith row
+
+            this->ueem[i] = this->dp / this->bee[0];
+            for(int j=1; j<=i; j++){ this->ueem[i] *= this->cee[j]/this->bee[j]; }
+
+          } else {
+
+            this->lee[i]  = 0.0;
+            this->leem[i] = 0.0;
+            this->uee[i]  = 0.0;
+            this->ueem[i] = 0.0;
+
+          }
+        }
+
+        {
+          Coeff_t delta_d = 1.0 / this->bee[0];
+          for(int j=1; j<Ls-1; j++){ delta_d *= this->cee[j] / this->bee[j]; }
+          this->dee[Ls-1] = this->bee[Ls-1] + this->cee[0] * this->dm * delta_d;
+          this->dee[Ls] = this->bee[Ls-1] + this->cee[Ls-1] * this->dp * delta_d;
+        }
+
+        int inv = 1;
+        this->MooeeInternalCompute(0, inv, this->MatpInv, this->MatmInv);
+        this->MooeeInternalCompute(1, inv, this->MatpInvDag, this->MatmInvDag);
+    }
+
+    // Recompute Cayley-form coefficients for different shift
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::RefreshShiftCoefficients(RealD new_shift)
+    {
+        this->shift = new_shift;
+        Approx::zolotarev_data *zdata = Approx::higham(1.0, this->Ls);
+        this->SetCoefficientsTanh(zdata, 1.0, 0.0);
+    }
+
+    template<class Impl>
+    void MobiusEOFAFermion<Impl>::MooeeInternalCompute(int dag, int inv,
+        Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm)
+    {
+        int Ls = this->Ls;
+
+        GridBase* grid = this->FermionRedBlackGrid();
+        int LLs = grid->_rdimensions[0];
+
+        if(LLs == Ls){
+            return; // Not vectorised in 5th direction
+        }
+
+        Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
+        Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);
+
+        for(int s=0; s<Ls; s++){
+            Pplus(s,s)  = this->bee[s];
+            Pminus(s,s) = this->bee[s];
+        }
+
+        for(int s=0; s<Ls-1; s++){
+            Pminus(s,s+1) = -this->cee[s];
+        }
+
+        for(int s=0; s<Ls-1; s++){
+            Pplus(s+1,s) = -this->cee[s+1];
+        }
+
+        Pplus (0,Ls-1) = this->dp;
+        Pminus(Ls-1,0) = this->dm;
+
+        Eigen::MatrixXcd PplusMat ;
+        Eigen::MatrixXcd PminusMat;
+
+        #if(0)
+            std::cout << GridLogMessage << "Pplus:" << std::endl;
+            for(int s=0; s<Ls; ++s){
+                for(int ss=0; ss<Ls; ++ss){
+                    std::cout << Pplus(s,ss) << "\t";
+                }
+                std::cout << std::endl;
+            }
+            std::cout << GridLogMessage << "Pminus:" << std::endl;
+            for(int s=0; s<Ls; ++s){
+                for(int ss=0; ss<Ls; ++ss){
+                    std::cout << Pminus(s,ss) << "\t";
+                }
+                std::cout << std::endl;
+            }
+        #endif
+
+        if(inv) {
+            PplusMat  = Pplus.inverse();
+            PminusMat = Pminus.inverse();
+        } else {
+            PplusMat  = Pplus;
+            PminusMat = Pminus;
+        }
+
+        if(dag){
+            PplusMat.adjointInPlace();
+            PminusMat.adjointInPlace();
+        }
+
+        typedef typename SiteHalfSpinor::scalar_type scalar_type;
+        const int Nsimd = Simd::Nsimd();
+        Matp.resize(Ls*LLs);
+        Matm.resize(Ls*LLs);
+
+        for(int s2=0; s2<Ls; s2++){
+        for(int s1=0; s1<LLs; s1++){
+            int istride = LLs;
+            int ostride = 1;
+            Simd Vp;
+            Simd Vm;
+            scalar_type *sp = (scalar_type*) &Vp;
+            scalar_type *sm = (scalar_type*) &Vm;
+            for(int l=0; l<Nsimd; l++){
+                if(switcheroo<Coeff_t>::iscomplex()) {
+                    sp[l] = PplusMat (l*istride+s1*ostride,s2);
+                    sm[l] = PminusMat(l*istride+s1*ostride,s2);
+                } else {
+                    // if real
+                    scalar_type tmp;
+                    tmp = PplusMat (l*istride+s1*ostride,s2);
+                    sp[l] = scalar_type(tmp.real(),tmp.real());
+                    tmp = PminusMat(l*istride+s1*ostride,s2);
+                    sm[l] = scalar_type(tmp.real(),tmp.real());
+                }
+            }
+            Matp[LLs*s2+s1] = Vp;
+            Matm[LLs*s2+s1] = Vm;
+        }}
+    }
+
+    FermOpTemplateInstantiate(MobiusEOFAFermion);
+    GparityFermOpTemplateInstantiate(MobiusEOFAFermion);
+
+}}
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermion.h b/lib/qcd/action/fermion/MobiusEOFAFermion.h
new file mode 100644
index 00000000..0a8d1788
--- /dev/null
+++ b/lib/qcd/action/fermion/MobiusEOFAFermion.h
@@ -0,0 +1,133 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/MobiusEOFAFermion.h
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#ifndef  GRID_QCD_MOBIUS_EOFA_FERMION_H
+#define  GRID_QCD_MOBIUS_EOFA_FERMION_H
+
+#include <Grid/qcd/action/fermion/AbstractEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+  template<class Impl>
+  class MobiusEOFAFermion : public AbstractEOFAFermion<Impl>
+  {
+    public:
+      INHERIT_IMPL_TYPES(Impl);
+
+    public:
+      // Shift operator coefficients for red-black preconditioned Mobius EOFA
+      std::vector<Coeff_t> Mooee_shift;
+      std::vector<Coeff_t> MooeeInv_shift_lc;
+      std::vector<Coeff_t> MooeeInv_shift_norm;
+      std::vector<Coeff_t> MooeeInvDag_shift_lc;
+      std::vector<Coeff_t> MooeeInvDag_shift_norm;
+
+      virtual void Instantiatable(void) {};
+
+      // EOFA-specific operations
+      virtual void  Omega            (const FermionField& in, FermionField& out, int sign, int dag);
+      virtual void  Dtilde           (const FermionField& in, FermionField& out);
+      virtual void  DtildeInv        (const FermionField& in, FermionField& out);
+
+      // override multiply
+      virtual RealD M                (const FermionField& in, FermionField& out);
+      virtual RealD Mdag             (const FermionField& in, FermionField& out);
+
+      // half checkerboard operations
+      virtual void  Mooee            (const FermionField& in, FermionField& out);
+      virtual void  MooeeDag         (const FermionField& in, FermionField& out);
+      virtual void  MooeeInv         (const FermionField& in, FermionField& out);
+      virtual void  MooeeInv_shift   (const FermionField& in, FermionField& out);
+      virtual void  MooeeInvDag      (const FermionField& in, FermionField& out);
+      virtual void  MooeeInvDag_shift(const FermionField& in, FermionField& out);
+
+      virtual void   M5D             (const FermionField& psi, FermionField& chi);
+      virtual void   M5Ddag          (const FermionField& psi, FermionField& chi);
+
+      /////////////////////////////////////////////////////
+      // Instantiate different versions depending on Impl
+      /////////////////////////////////////////////////////
+      void M5D(const FermionField& psi, const FermionField& phi, FermionField& chi,
+        std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper);
+
+      void M5D_shift(const FermionField& psi, const FermionField& phi, FermionField& chi,
+        std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper,
+        std::vector<Coeff_t>& shift_coeffs);
+
+      void M5Ddag(const FermionField& psi, const FermionField& phi, FermionField& chi,
+        std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper);
+
+      void M5Ddag_shift(const FermionField& psi, const FermionField& phi, FermionField& chi,
+        std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper,
+        std::vector<Coeff_t>& shift_coeffs);
+
+      void MooeeInternal(const FermionField& in, FermionField& out, int dag, int inv);
+
+      void MooeeInternalCompute(int dag, int inv, Vector<iSinglet<Simd>>& Matp, Vector<iSinglet<Simd>>& Matm);
+
+      void MooeeInternalAsm(const FermionField& in, FermionField& out, int LLs, int site,
+        Vector<iSinglet<Simd>>& Matp, Vector<iSinglet<Simd>>& Matm);
+
+      void MooeeInternalZAsm(const FermionField& in, FermionField& out, int LLs, int site,
+        Vector<iSinglet<Simd>>& Matp, Vector<iSinglet<Simd>>& Matm);
+
+      virtual void RefreshShiftCoefficients(RealD new_shift);
+
+      // Constructors
+      MobiusEOFAFermion(GaugeField& _Umu, GridCartesian& FiveDimGrid, GridRedBlackCartesian& FiveDimRedBlackGrid,
+        GridCartesian& FourDimGrid, GridRedBlackCartesian& FourDimRedBlackGrid,
+        RealD _mq1, RealD _mq2, RealD _mq3, RealD _shift, int pm,
+        RealD _M5, RealD _b, RealD _c, const ImplParams& p=ImplParams());
+
+    protected:
+      void SetCoefficientsPrecondShiftOps(void);
+  };
+}}
+
+#define INSTANTIATE_DPERP_DWF_EOFA(A)\
+template void MobiusEOFAFermion<A>::M5D(const FermionField& psi, const FermionField& phi, FermionField& chi, \
+  std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper); \
+template void MobiusEOFAFermion<A>::M5D_shift(const FermionField& psi, const FermionField& phi, FermionField& chi, \
+  std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper, std::vector<Coeff_t>& shift_coeffs); \
+template void MobiusEOFAFermion<A>::M5Ddag(const FermionField& psi, const FermionField& phi, FermionField& chi, \
+  std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper); \
+template void MobiusEOFAFermion<A>::M5Ddag_shift(const FermionField& psi, const FermionField& phi, FermionField& chi, \
+  std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper, std::vector<Coeff_t>& shift_coeffs); \
+template void MobiusEOFAFermion<A>::MooeeInv(const FermionField& psi, FermionField& chi); \
+template void MobiusEOFAFermion<A>::MooeeInv_shift(const FermionField& psi, FermionField& chi); \
+template void MobiusEOFAFermion<A>::MooeeInvDag(const FermionField& psi, FermionField& chi);
+template void MobiusEOFAFermion<A>::MooeeInvDag_shift(const FermionField& psi, FermionField& chi);
+
+#undef  MOBIUS_EOFA_DPERP_DENSE
+#define MOBIUS_EOFA_DPERP_CACHE
+#undef  MOBIUS_EOFA_DPERP_LINALG
+#define MOBIUS_EOFA_DPERP_VEC
+
+#endif

From ac9e6b63c092dedde5af757a7b0fcd7b182f1523 Mon Sep 17 00:00:00 2001
From: David Murphy <david.murphy@ligo.org>
Date: Thu, 17 Aug 2017 19:28:53 -0400
Subject: [PATCH 041/115] More re-import of Mobius EOFA

---
 .../action/fermion/DomainWallEOFAFermion.cc   |   4 +-
 lib/qcd/action/fermion/MobiusEOFAFermion.cc   | 486 ++++++++++--------
 2 files changed, 261 insertions(+), 229 deletions(-)

diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermion.cc b/lib/qcd/action/fermion/DomainWallEOFAFermion.cc
index dc4f6504..dd8a500d 100644
--- a/lib/qcd/action/fermion/DomainWallEOFAFermion.cc
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermion.cc
@@ -347,9 +347,7 @@ namespace QCD {
         GridBase* grid = this->FermionRedBlackGrid();
         int LLs = grid->_rdimensions[0];
 
-        if(LLs == Ls){
-            return; // Not vectorised in 5th direction
-        }
+        if(LLs == Ls){ return; } // Not vectorised in 5th direction
 
         Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
         Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermion.cc b/lib/qcd/action/fermion/MobiusEOFAFermion.cc
index 9d0d8b11..c0837d78 100644
--- a/lib/qcd/action/fermion/MobiusEOFAFermion.cc
+++ b/lib/qcd/action/fermion/MobiusEOFAFermion.cc
@@ -133,7 +133,36 @@ namespace QCD {
     }
 
     template<class Impl>
-    void MobiusEOFAFermion<Impl>::DtildeInv(const FermionField& psi, FermionField& chi){ }
+    void MobiusEOFAFermion<Impl>::DtildeInv(const FermionField& psi, FermionField& chi)
+    {
+      int Ls = this->Ls;
+      RealD m = this->mq1;
+      RealD c = 0.5 * this->alpha;
+      RealD d = 0.5;
+
+      RealD DtInv_p(0.0), DtInv_m(0.0);
+      RealD N = std::pow(c+d,Ls) + m*std::pow(c-d,Ls);
+      FermionField tmp = zero;
+
+      for(int s=0; s<Ls; ++s){
+      for(int sp=0; sp<Ls; ++sp){
+
+        DtInv_p = m * std::pow(-1.0,s-sp+1) * std::pow(c-d,Ls+s-sp) / std::pow(c+d,s-sp+1) / N;
+        DtInv_p += (s < sp) ? 0.0 : std::pow(-1.0,s-sp) * std::pow(c-d,s-sp) / std::pow(c+d,s-sp+1);
+        DtInv_m = m * std::pow(-1.0,sp-s+1) * std::pow(c-d,Ls+sp-s) / std::pow(c+d,sp-s+1) / N;
+        DtInv_m += (s > sp) ? 0.0 : std::pow(-1.0,sp-s) * std::pow(c-d,sp-s) / std::pow(c+d,sp-s+1);
+
+        if(dag){
+          RealD tmp(DtInv_p);
+          DtInv_p = DtInv_m;
+          DtInv_m = tmp;
+        }
+
+        axpby_ssp_pplus (tmp, 1.0, tmp, DtInv_p, psi, s, sp);
+        axpby_ssp_pminus(tmp, 1.0, tmp, DtInv_m, psi, s, sp);
+
+      }}
+    }
 
     /*****************************************************************************************************/
 
@@ -172,298 +201,303 @@ namespace QCD {
     template<class Impl>
     void MobiusEOFAFermion<Impl>::M5D(const FermionField& psi, FermionField& chi)
     {
-        int Ls = this->Ls;
+      int Ls = this->Ls;
 
-        std::vector<Coeff_t> diag(Ls,1.0);
-        std::vector<Coeff_t> upper(Ls,-1.0);  upper[Ls-1] = this->mq1;
-        std::vector<Coeff_t> lower(Ls,-1.0);  lower[0]    = this->mq1;
+      std::vector<Coeff_t> diag(Ls,1.0);
+      std::vector<Coeff_t> upper(Ls,-1.0);  upper[Ls-1] = this->mq1;
+      std::vector<Coeff_t> lower(Ls,-1.0);  lower[0]    = this->mq1;
 
-        // no shift term
-        if(this->shift == 0.0){ this->M5D(psi, chi, chi, lower, diag, upper); }
+      // no shift term
+      if(this->shift == 0.0){ this->M5D(psi, chi, chi, lower, diag, upper); }
 
-        // fused M + shift operation
-        else{ this->M5D_shift(psi, chi, chi, lower, diag, upper, Mooee_shift); }
+      // fused M + shift operation
+      else{ this->M5D_shift(psi, chi, chi, lower, diag, upper, Mooee_shift); }
     }
 
     template<class Impl>
     void MobiusEOFAFermion<Impl>::M5Ddag(const FermionField& psi, FermionField& chi)
     {
-        int   Ls    = this->Ls;
-        int   pm    = this->pm;
-        RealD shift = this->shift;
-        RealD mq1   = this->mq1;
-        RealD mq2   = this->mq2;
-        RealD mq3   = this->mq3;
+      int   Ls    = this->Ls;
 
-        // coefficients for shift operator ( = shift*\gamma_{5}*R_{5}*\Delta_{\pm}(mq2,mq3)*P_{\pm} )
-        Coeff_t shiftp(0.0), shiftm(0.0);
-        if(shift != 0.0){
-          if(pm == 1){ shiftp = shift*(mq3-mq2); }
-          else{ shiftm = -shift*(mq3-mq2); }
-        }
+      std::vector<Coeff_t> diag(Ls,1.0);
+      std::vector<Coeff_t> upper(Ls,-1.0); upper[Ls-1] = this->mq1 + shiftp;
+      std::vector<Coeff_t> lower(Ls,-1.0); lower[0]    = this->mq1 + shiftm;
 
-        std::vector<Coeff_t> diag(Ls,1.0);
-        std::vector<Coeff_t> upper(Ls,-1.0); upper[Ls-1] = mq1 + shiftp;
-        std::vector<Coeff_t> lower(Ls,-1.0); lower[0]    = mq1 + shiftm;
+      // no shift term
+      if(this->shift == 0.0){ this->M5Ddag(psi, chi, chi, lower, diag, upper); }
 
-        #if(0)
-            std::cout << GridLogMessage << "MobiusEOFAFermion::M5Ddag(FF&,FF&):" << std::endl;
-            for(int i=0; i<diag.size(); ++i){
-                std::cout << GridLogMessage << "diag[" << i << "] =" << diag[i] << std::endl;
-            }
-            for(int i=0; i<upper.size(); ++i){
-                std::cout << GridLogMessage << "upper[" << i << "] =" << upper[i] << std::endl;
-            }
-            for(int i=0; i<lower.size(); ++i){
-                std::cout << GridLogMessage << "lower[" << i << "] =" << lower[i] << std::endl;
-            }
-        #endif
-
-        this->M5Ddag(psi, chi, chi, lower, diag, upper);
+      // fused M + shift operation
+      else{ this->M5Ddag_shift(psi, chi, chi, lower, diag, upper, Mooee_shift); }
     }
 
     // half checkerboard operations
     template<class Impl>
     void MobiusEOFAFermion<Impl>::Mooee(const FermionField& psi, FermionField& chi)
     {
-        int Ls = this->Ls;
+      int Ls = this->Ls;
 
-        std::vector<Coeff_t> diag = this->bee;
-        std::vector<Coeff_t> upper(Ls);
-        std::vector<Coeff_t> lower(Ls);
+      // coefficients of Mooee
+      std::vector<Coeff_t> diag = this->bee;
+      std::vector<Coeff_t> upper(Ls);
+      std::vector<Coeff_t> lower(Ls);
+      for(int s=0; s<Ls; s++){
+        upper[s] = -this->cee[s];
+        lower[s] = -this->cee[s];
+      }
+      upper[Ls-1] *= -this->mq1;
+      lower[0]    *= -this->mq1;
 
-        for(int s=0; s<Ls; s++){
-          upper[s] = -this->cee[s];
-          lower[s] = -this->cee[s];
-        }
-        upper[Ls-1] = this->dm;
-        lower[0]    = this->dp;
+      // no shift term
+      if(this->shift == 0.0){ this->M5D(psi, psi, chi, lower, diag, upper); }
 
-        this->M5D(psi, psi, chi, lower, diag, upper);
+      // fused M + shift operation
+      else { this->M5D_shift(psi, psi, chi, lower, diag, upper, Mooee_shift); }
     }
 
     template<class Impl>
     void MobiusEOFAFermion<Impl>::MooeeDag(const FermionField& psi, FermionField& chi)
     {
-        int Ls = this->Ls;
+      int Ls = this->Ls;
 
-        std::vector<Coeff_t> diag = this->bee;
-        std::vector<Coeff_t> upper(Ls);
-        std::vector<Coeff_t> lower(Ls);
-
-        for(int s=0; s<Ls; s++){
-          upper[s] = -this->cee[s];
-          lower[s] = -this->cee[s];
+      // coefficients of MooeeDag
+      std::vector<Coeff_t> diag = this->bee;
+      std::vector<Coeff_t> upper(Ls);
+      std::vector<Coeff_t> lower(Ls);
+      for(int s=0; s<Ls; s++){
+        if(s==0) {
+          upper[s] = -this->cee[s+1];
+          lower[s] = this->mq1*this->cee[Ls-1];
+        } else if(s==(Ls-1)) {
+          upper[s] = this->mq1*this->cee[0];
+          lower[s] = -this->cee[s-1];
+        } else {
+          upper[s] = -this->cee[s+1];
+          lower[s] = -this->cee[s-1];
         }
-        upper[Ls-1] = this->dp;
-        lower[0]    = this->dm;
+      }
 
-        this->M5Ddag(psi, psi, chi, lower, diag, upper);
+      // no shift term
+      if(this->shift == 0.0){ this->M5Ddag(psi, psi, chi, lower, diag, upper); }
+
+      // fused M + shift operation
+      else{ this->M5Ddag_shift(psi, psi, chi, lower, diag, upper, Mooee_shift); }
     }
 
     /****************************************************************************************/
 
-    //Zolo
+    // Computes coefficients for applying Cayley preconditioned shift operators
+    //  (Mooee + \Delta) --> Mooee_shift
+    //  (Mooee + \Delta)^{-1} --> MooeeInv_shift_lc, MooeeInv_shift_norm
+    //  (Mooee + \Delta)^{-dag} --> MooeeInvDag_shift_lc, MooeeInvDag_shift_norm
+    // For the latter two cases, the operation takes the form
+    //  [ (Mooee + \Delta)^{-1} \psi ]_{i} = Mooee_{ij} \psi_{j} +
+    //      ( MooeeInv_shift_norm )_{i} ( \sum_{j} [ MooeeInv_shift_lc ]_{j} P_{pm} \psi_{j} )
     template<class Impl>
-    void MobiusEOFAFermion<Impl>::SetCoefficientsInternal(RealD zolo_hi, std::vector<Coeff_t>& gamma, RealD b, RealD c)
+    void MobiusEOFAFermion<Impl>::SetCoefficientsPrecondShiftOps()
     {
-        int   Ls    = this->Ls;
-        int   pm    = this->pm;
-        RealD mq1   = this->mq1;
-        RealD mq2   = this->mq2;
-        RealD mq3   = this->mq3;
-        RealD shift = this->shift;
+      int   Ls    = this->Ls;
+      int   pm    = this->pm;
+      RealD alpha = this->alpha;
+      RealD k     = this->k;
+      RealD mq1   = this->mq1;
+      RealD shift = this->shift;
 
-        ////////////////////////////////////////////////////////
-        // Constants for the preconditioned matrix Cayley form
-        ////////////////////////////////////////////////////////
-        this->bs.resize(Ls);
-        this->cs.resize(Ls);
-        this->aee.resize(Ls);
-        this->aeo.resize(Ls);
-        this->bee.resize(Ls);
-        this->beo.resize(Ls);
-        this->cee.resize(Ls);
-        this->ceo.resize(Ls);
+      // Initialize
+      Mooee_shift.resize(Ls);
+      MooeeInv_shift_lc.resize(Ls);
+      MooeeInv_shift_norm.resize(Ls);
+      MooeeInvDag_shift_lc.resize(Ls);
+      MooeeInvDag_shift_norm.resize(Ls);
 
-        for(int i=0; i<Ls; ++i){
-          this->bee[i] = 4.0 - this->M5 + 1.0;
-          this->cee[i] = 1.0;
-        }
+      // Construct Mooee_shift
+      int idx(0);
+      Coeff_t N = ( (pm == 1) ? 1.0 : -1.0 ) * (2.0*shift*k) *
+                  ( std::pow(alpha+1.0,Ls) + mq1*std::pow(alpha-1.0,Ls) );
+      for(int s=0; s<Ls; ++s){
+        idx = (pm == 1) ? (s) : (Ls-1-s);
+        Mooee_shift[idx] = N * std::pow(-1.0,s) * std::pow(alpha-1.0,s) / std::pow(alpha+1.0,Ls+s+1);
+      }
 
-        for(int i=0; i<Ls; ++i){
-          this->aee[i] = this->cee[i];
-          this->bs[i] = this->beo[i] = 1.0;
-          this->cs[i] = this->ceo[i] = 0.0;
-        }
+      // Tridiagonal solve for MooeeInvDag_shift_lc
+      {
+        Coeff_t m(0.0);
+        std::vector<Coeff_t> d = Mooee_shift;
+        std::vector<Coeff_t> u(Ls,0.0);
+        std::vector<Coeff_t> y(Ls,0.0);
+        std::vector<Coeff_t> q(Ls,0.0);
+        if(pm == 1){ u[0] = 1.0; }
+        else{ u[Ls-1] = 1.0; }
 
-        //////////////////////////////////////////
-        // EOFA shift terms
-        //////////////////////////////////////////
+        // Tridiagonal matrix algorithm + Sherman-Morrison formula
+        //
+        // We solve
+        //  ( Mooee' + u \otimes v ) MooeeInvDag_shift_lc = Mooee_shift
+        // where Mooee' is the tridiagonal part of Mooee_{+}, and
+        // u = (1,0,...,0) and v = (0,...,0,mq1*cee[0]) are chosen
+        // so that the outer-product u \otimes v gives the (0,Ls-1)
+        // entry of Mooee_{+}.
+        //
+        // We do this as two solves: Mooee'*y = d and Mooee'*q = u,
+        // and then construct the solution to the original system
+        //  MooeeInvDag_shift_lc = y - <v,y> / ( 1 + <v,q> ) q
         if(pm == 1){
-          this->dp = mq1*this->cee[0] + shift*(mq3-mq2);
-          this->dm = mq1*this->cee[Ls-1];
-        } else if(this->pm == -1) {
-          this->dp = mq1*this->cee[0];
-          this->dm = mq1*this->cee[Ls-1] - shift*(mq3-mq2);
-        } else {
-          this->dp = mq1*this->cee[0];
-          this->dm = mq1*this->cee[Ls-1];
+          for(int s=1; s<Ls; ++s){
+            m = -this->cee[s] / this->bee[s-1];
+            d[s] -= m*d[s-1];
+            u[s] -= m*u[s-1];
+          }
         }
-
-        //////////////////////////////////////////
-        // LDU decomposition of eeoo
-        //////////////////////////////////////////
-        this->dee.resize(Ls+1);
-        this->lee.resize(Ls);
-        this->leem.resize(Ls);
-        this->uee.resize(Ls);
-        this->ueem.resize(Ls);
-
-        for(int i=0; i<Ls; ++i){
-
-          if(i < Ls-1){
-
-            this->lee[i] = -this->cee[i+1]/this->bee[i]; // sub-diag entry on the ith column
-
-            this->leem[i] = this->dm/this->bee[i];
-            for(int j=0; j<i; j++){ this->leem[i] *= this->aee[j]/this->bee[j]; }
-
-            this->dee[i] = this->bee[i];
-
-            this->uee[i] = -this->aee[i]/this->bee[i];   // up-diag entry on the ith row
-
-            this->ueem[i] = this->dp / this->bee[0];
-            for(int j=1; j<=i; j++){ this->ueem[i] *= this->cee[j]/this->bee[j]; }
-
+        y[Ls-1] = d[Ls-1] / this->bee[Ls-1];
+        q[Ls-1] = u[Ls-1] / this->bee[Ls-1];
+        for(int s=Ls-2; s>=0; --s){
+          if(pm == 1){
+            y[s] = d[s] / this->bee[s];
+            q[s] = u[s] / this->bee[s];
           } else {
-
-            this->lee[i]  = 0.0;
-            this->leem[i] = 0.0;
-            this->uee[i]  = 0.0;
-            this->ueem[i] = 0.0;
-
+            y[s] = ( d[s] + this->cee[s]*y[s+1] ) / this->bee[s];
+            q[s] = ( u[s] + this->cee[s]*q[s+1] ) / this->bee[s];
           }
         }
 
-        {
-          Coeff_t delta_d = 1.0 / this->bee[0];
-          for(int j=1; j<Ls-1; j++){ delta_d *= this->cee[j] / this->bee[j]; }
-          this->dee[Ls-1] = this->bee[Ls-1] + this->cee[0] * this->dm * delta_d;
-          this->dee[Ls] = this->bee[Ls-1] + this->cee[Ls-1] * this->dp * delta_d;
+        // Construct MooeeInvDag_shift_lc
+        for(int s=0; s<Ls; ++s){
+          if(pm == 1){
+            MooeeInvDag_shift_lc[s] = y[s] - mq1*this->cee[0]*y[Ls-1] /
+              (1.0+mq1*this->cee[0]*q[Ls-1]) * q[s];
+          } else {
+            MooeeInvDag_shift_lc[s] = y[s] - mq1*this->cee[Ls-1]*y[0] /
+              (1.0+mq1*this->cee[Ls-1]*q[0]) * q[s];
+          }
         }
 
-        int inv = 1;
-        this->MooeeInternalCompute(0, inv, this->MatpInv, this->MatmInv);
-        this->MooeeInternalCompute(1, inv, this->MatpInvDag, this->MatmInvDag);
+        // Compute remaining coefficients
+        N = (pm == 1) ? (1.0 + MooeeInvDag_shift_lc[Ls-1]) : (1.0 + MooeeInvDag_shift_lc[0]);
+        for(int s=0; s<Ls; ++s){
+
+          // MooeeInv_shift_lc
+          if(pm == 1){ MooeeInv_shift_lc[s] = std::pow(this->bee[s],s) * std::pow(this->cee[s],Ls-1-s); }
+          else{ MooeeInv_shift_lc[s] = std::pow(this->bee[s],Ls-1-s) * std::pow(this->cee[s],s); }
+
+          // MooeeInv_shift_norm
+          MooeeInv_shift_norm[s] = -MooeeInvDag_shift_lc[s] /
+            ( std::pow(this->bee[s],Ls) + mq1*std::pow(this->cee[s],Ls) ) / N;
+
+          // MooeeInvDag_shift_norm
+          if(pm == 1){ MooeeInvDag_shift_norm[s] = -std::pow(this->bee[s],s) * std::pow(this->cee[s],Ls-1-s) /
+            ( std::pow(this->bee[s],Ls) + mq1*std::pow(this->cee[s],Ls) ) / N; }
+          else{ MooeeInvDag_shift_norm[s] = -std::pow(this->bee[s],Ls-1-s) * std::pow(this->cee[s],s) /
+            ( std::pow(this->bee[s],Ls) + mq1*std::pow(this->cee[s],Ls) ) / N; }
+        }
+      }
     }
 
-    // Recompute Cayley-form coefficients for different shift
+    // Recompute coefficients for a different value of shift constant
     template<class Impl>
     void MobiusEOFAFermion<Impl>::RefreshShiftCoefficients(RealD new_shift)
     {
-        this->shift = new_shift;
-        Approx::zolotarev_data *zdata = Approx::higham(1.0, this->Ls);
-        this->SetCoefficientsTanh(zdata, 1.0, 0.0);
+      this->shift = new_shift;
+      if(new_shift != 0.0){
+        SetCoefficientsPrecondShiftOps();
+      } else {
+        int Ls = this->Ls;
+        Mooee_shift.resize(Ls,0.0);
+        MooeeInv_shift_lc.resize(Ls,0.0);
+        MooeeInv_shift_norm.resize(Ls,0.0);
+        MooeeInvDag_shift_lc.resize(Ls,0.0);
+        MooeeInvDag_shift_norm.resize(Ls,0.0);
+      }
     }
 
     template<class Impl>
     void MobiusEOFAFermion<Impl>::MooeeInternalCompute(int dag, int inv,
-        Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm)
+      Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm)
     {
-        int Ls = this->Ls;
+      int Ls = this->Ls;
 
-        GridBase* grid = this->FermionRedBlackGrid();
-        int LLs = grid->_rdimensions[0];
+      GridBase* grid = this->FermionRedBlackGrid();
+      int LLs = grid->_rdimensions[0];
 
-        if(LLs == Ls){
-            return; // Not vectorised in 5th direction
-        }
+      if(LLs == Ls){ return; } // Not vectorised in 5th direction
 
-        Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
-        Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);
+      Eigen::MatrixXcd Pplus  = Eigen::MatrixXcd::Zero(Ls,Ls);
+      Eigen::MatrixXcd Pminus = Eigen::MatrixXcd::Zero(Ls,Ls);
 
-        for(int s=0; s<Ls; s++){
-            Pplus(s,s)  = this->bee[s];
-            Pminus(s,s) = this->bee[s];
-        }
+      for(int s=0; s<Ls; s++){
+        Pplus(s,s)  = this->bee[s];
+        Pminus(s,s) = this->bee[s];
+      }
 
-        for(int s=0; s<Ls-1; s++){
-            Pminus(s,s+1) = -this->cee[s];
-        }
+      for(int s=0; s<Ls-1; s++){
+        Pminus(s,s+1) = -this->cee[s];
+        Pplus(s+1,s) = -this->cee[s+1];
+      }
 
-        for(int s=0; s<Ls-1; s++){
-            Pplus(s+1,s) = -this->cee[s+1];
-        }
+      Pplus (0,Ls-1) = this->mq1*this->cee[0];
+      Pminus(Ls-1,0) = this->mq1*this->cee[Ls-1];
 
-        Pplus (0,Ls-1) = this->dp;
-        Pminus(Ls-1,0) = this->dm;
-
-        Eigen::MatrixXcd PplusMat ;
-        Eigen::MatrixXcd PminusMat;
-
-        #if(0)
-            std::cout << GridLogMessage << "Pplus:" << std::endl;
-            for(int s=0; s<Ls; ++s){
-                for(int ss=0; ss<Ls; ++ss){
-                    std::cout << Pplus(s,ss) << "\t";
-                }
-                std::cout << std::endl;
-            }
-            std::cout << GridLogMessage << "Pminus:" << std::endl;
-            for(int s=0; s<Ls; ++s){
-                for(int ss=0; ss<Ls; ++ss){
-                    std::cout << Pminus(s,ss) << "\t";
-                }
-                std::cout << std::endl;
-            }
-        #endif
-
-        if(inv) {
-            PplusMat  = Pplus.inverse();
-            PminusMat = Pminus.inverse();
+      if(this->shift != 0.0){
+        RealD c = 0.5 * this->alpha;
+        RealD d = 0.5;
+        RealD N = this->shift * this->k * ( std::pow(c+d,Ls) + this->mq1*std::pow(c-d,Ls) );
+        if(this->pm == 1) {
+          for(int s=0; s<Ls; ++s){
+            Pplus(s,Ls-1) += N * std::pow(-1.0,s) * std::pow(c-d,s) / std::pow(c+d,Ls+s+1);
+          }
         } else {
-            PplusMat  = Pplus;
-            PminusMat = Pminus;
+          for(int s=0; s<Ls; ++s){
+            Pminus(s,0) += N * std::pow(-1.0,s+1) * std::pow(c-d,Ls-1-s) / std::pow(c+d,2*Ls-s);
+          }
         }
+      }
 
-        if(dag){
-            PplusMat.adjointInPlace();
-            PminusMat.adjointInPlace();
+      Eigen::MatrixXcd PplusMat ;
+      Eigen::MatrixXcd PminusMat;
+
+      if(inv) {
+        PplusMat  = Pplus.inverse();
+        PminusMat = Pminus.inverse();
+      } else {
+        PplusMat  = Pplus;
+        PminusMat = Pminus;
+      }
+
+      if(dag){
+        PplusMat.adjointInPlace();
+        PminusMat.adjointInPlace();
+      }
+
+      typedef typename SiteHalfSpinor::scalar_type scalar_type;
+      const int Nsimd = Simd::Nsimd();
+      Matp.resize(Ls*LLs);
+      Matm.resize(Ls*LLs);
+
+      for(int s2=0; s2<Ls; s2++){
+      for(int s1=0; s1<LLs; s1++){
+        int istride = LLs;
+        int ostride = 1;
+        Simd Vp;
+        Simd Vm;
+        scalar_type *sp = (scalar_type*) &Vp;
+        scalar_type *sm = (scalar_type*) &Vm;
+        for(int l=0; l<Nsimd; l++){
+          if(switcheroo<Coeff_t>::iscomplex()) {
+            sp[l] = PplusMat (l*istride+s1*ostride,s2);
+            sm[l] = PminusMat(l*istride+s1*ostride,s2);
+          } else {
+            // if real
+            scalar_type tmp;
+            tmp = PplusMat (l*istride+s1*ostride,s2);
+            sp[l] = scalar_type(tmp.real(),tmp.real());
+            tmp = PminusMat(l*istride+s1*ostride,s2);
+            sm[l] = scalar_type(tmp.real(),tmp.real());
+          }
         }
+        Matp[LLs*s2+s1] = Vp;
+        Matm[LLs*s2+s1] = Vm;
+      }}
+  }
 
-        typedef typename SiteHalfSpinor::scalar_type scalar_type;
-        const int Nsimd = Simd::Nsimd();
-        Matp.resize(Ls*LLs);
-        Matm.resize(Ls*LLs);
-
-        for(int s2=0; s2<Ls; s2++){
-        for(int s1=0; s1<LLs; s1++){
-            int istride = LLs;
-            int ostride = 1;
-            Simd Vp;
-            Simd Vm;
-            scalar_type *sp = (scalar_type*) &Vp;
-            scalar_type *sm = (scalar_type*) &Vm;
-            for(int l=0; l<Nsimd; l++){
-                if(switcheroo<Coeff_t>::iscomplex()) {
-                    sp[l] = PplusMat (l*istride+s1*ostride,s2);
-                    sm[l] = PminusMat(l*istride+s1*ostride,s2);
-                } else {
-                    // if real
-                    scalar_type tmp;
-                    tmp = PplusMat (l*istride+s1*ostride,s2);
-                    sp[l] = scalar_type(tmp.real(),tmp.real());
-                    tmp = PminusMat(l*istride+s1*ostride,s2);
-                    sm[l] = scalar_type(tmp.real(),tmp.real());
-                }
-            }
-            Matp[LLs*s2+s1] = Vp;
-            Matm[LLs*s2+s1] = Vm;
-        }}
-    }
-
-    FermOpTemplateInstantiate(MobiusEOFAFermion);
-    GparityFermOpTemplateInstantiate(MobiusEOFAFermion);
+  FermOpTemplateInstantiate(MobiusEOFAFermion);
+  GparityFermOpTemplateInstantiate(MobiusEOFAFermion);
 
 }}

From 9d45fca8bc11caf55cbbaebb290c8786cc6d34b8 Mon Sep 17 00:00:00 2001
From: David Murphy <djm2131@columbia.edu>
Date: Thu, 17 Aug 2017 23:45:36 -0400
Subject: [PATCH 042/115] Implement MobiusEOFAFermioncache.cc

---
 .../action/fermion/MobiusEOFAFermioncache.cc  | 429 ++++++++++++++++++
 1 file changed, 429 insertions(+)
 create mode 100644 lib/qcd/action/fermion/MobiusEOFAFermioncache.cc

diff --git a/lib/qcd/action/fermion/MobiusEOFAFermioncache.cc b/lib/qcd/action/fermion/MobiusEOFAFermioncache.cc
new file mode 100644
index 00000000..d184ebe2
--- /dev/null
+++ b/lib/qcd/action/fermion/MobiusEOFAFermioncache.cc
@@ -0,0 +1,429 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/MobiusEOFAFermioncache.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/MobiusEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+  // FIXME -- make a version of these routines with site loop outermost for cache reuse.
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5D(const FermionField &psi, const FermionField &phi, FermionField &chi,
+    std::vector<Coeff_t> &lower, std::vector<Coeff_t> &diag, std::vector<Coeff_t> &upper)
+  {
+    int Ls = this->Ls;
+    GridBase *grid = psi._grid;
+
+    assert(phi.checkerboard == psi.checkerboard);
+    chi.checkerboard = psi.checkerboard;
+
+    // Flops = 6.0*(Nc*Ns) *Ls*vol
+    this->M5Dcalls++;
+    this->M5Dtime -= usecond();
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){
+      for(int s=0; s<Ls; s++){
+        auto tmp = psi._odata[0];
+        if(s==0){
+          spProj5m(tmp, psi._odata[ss+s+1]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5p(tmp, psi._odata[ss+Ls-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        } else if(s==(Ls-1)) {
+          spProj5m(tmp, psi._odata[ss+0]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5p(tmp, psi._odata[ss+s-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        } else {
+          spProj5m(tmp, psi._odata[ss+s+1]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5p(tmp, psi._odata[ss+s-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        }
+      }
+    }
+
+    this->M5Dtime += usecond();
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5D_shift(const FermionField &psi, const FermionField &phi, FermionField &chi,
+    std::vector<Coeff_t> &lower, std::vector<Coeff_t> &diag, std::vector<Coeff_t> &upper,
+    std::vector<Coeff_t> &shift_coeffs)
+  {
+    int Ls = this->Ls;
+    int shift_s = (this->pm == 1) ? (Ls-1) : 0; // s-component modified by shift operator
+    GridBase *grid = psi._grid;
+
+    assert(phi.checkerboard == psi.checkerboard);
+    chi.checkerboard = psi.checkerboard;
+
+    // Flops = 6.0*(Nc*Ns) *Ls*vol
+    this->M5Dcalls++;
+    this->M5Dtime -= usecond();
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){
+      for(int s=0; s<Ls; s++){
+        auto tmp = psi._odata[0];
+        if(s==0){
+          spProj5m(tmp, psi._odata[ss+s+1]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5p(tmp, psi._odata[ss+Ls-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        } else if(s==(Ls-1)) {
+          spProj5m(tmp, psi._odata[ss+0]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5p(tmp, psi._odata[ss+s-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        } else {
+          spProj5m(tmp, psi._odata[ss+s+1]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5p(tmp, psi._odata[ss+s-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        }
+        if(this->pm == 1){ spProj5p(tmp, psi._odata[ss+shift_s]); }
+        else{ spProj5m(tmp, psi._odata[ss+shift_s]); }
+        chi[ss+s] = chi[ss+s] + shift_coeffs[s]*tmp;
+      }
+    }
+
+    this->M5Dtime += usecond();
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5Ddag(const FermionField &psi, const FermionField &phi, FermionField &chi,
+    std::vector<Coeff_t> &lower, std::vector<Coeff_t> &diag, std::vector<Coeff_t> &upper)
+  {
+    int Ls = this->Ls;
+    GridBase *grid = psi._grid;
+
+    assert(phi.checkerboard == psi.checkerboard);
+    chi.checkerboard = psi.checkerboard;
+
+    // Flops = 6.0*(Nc*Ns) *Ls*vol
+    this->M5Dcalls++;
+    this->M5Dtime -= usecond();
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){
+      auto tmp = psi._odata[0];
+      for(int s=0; s<Ls; s++){
+        if(s==0) {
+          spProj5p(tmp, psi._odata[ss+s+1]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5m(tmp, psi._odata[ss+Ls-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        } else if(s==(Ls-1)) {
+          spProj5p(tmp, psi._odata[ss+0]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5m(tmp, psi._odata[ss+s-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        } else {
+          spProj5p(tmp, psi._odata[ss+s+1]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5m(tmp, psi._odata[ss+s-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        }
+      }
+    }
+
+    this->M5Dtime += usecond();
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5Ddag_shift(const FermionField &psi, const FermionField &phi, FermionField &chi,
+    std::vector<Coeff_t> &lower, std::vector<Coeff_t> &diag, std::vector<Coeff_t> &upper,
+    std::vector<Coeff_t> &shift_coeffs)
+  {
+    int Ls = this->Ls;
+    int shift_s = (this->pm == 1) ? (Ls-1) : 0; // s-component modified by shift operator
+    GridBase *grid = psi._grid;
+
+    assert(phi.checkerboard == psi.checkerboard);
+    chi.checkerboard = psi.checkerboard;
+
+    // Flops = 6.0*(Nc*Ns) *Ls*vol
+    this->M5Dcalls++;
+    this->M5Dtime -= usecond();
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){
+      chi[ss+Ls-1] = zero;
+      auto tmp = psi._odata[0];
+      for(int s=0; s<Ls; s++){
+        if(s==0) {
+          spProj5p(tmp, psi._odata[ss+s+1]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5m(tmp, psi._odata[ss+Ls-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        } else if(s==(Ls-1)) {
+          spProj5p(tmp, psi._odata[ss+0]);
+          chi[ss+s] = chi[ss+s] + diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5m(tmp, psi._odata[ss+s-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        } else {
+          spProj5p(tmp, psi._odata[ss+s+1]);
+          chi[ss+s] = diag[s]*phi[ss+s] + upper[s]*tmp;
+          spProj5m(tmp, psi._odata[ss+s-1]);
+          chi[ss+s] = chi[ss+s] + lower[s]*tmp;
+        }
+        if(this->pm == 1){ spProj5p(tmp, psi._odata[ss+s]); }
+        else{ spProj5m(tmp, psi._odata[ss+s]); }
+        chi[ss+shift_s] = chi[ss+shift_s] + shift_coeffs[s]*tmp;
+      }
+    }
+
+    this->M5Dtime += usecond();
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInv(const FermionField &psi, FermionField &chi)
+  {
+    if(this->shift != 0.0){ MooeeInv_shift(psi,chi); return; }
+
+    GridBase *grid = psi._grid;
+    int Ls = this->Ls;
+
+    chi.checkerboard = psi.checkerboard;
+
+    this->MooeeInvCalls++;
+    this->MooeeInvTime -= usecond();
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){
+
+      auto tmp = psi._odata[0];
+
+      // Apply (L^{\prime})^{-1}
+      chi[ss] = psi[ss]; // chi[0]=psi[0]
+      for(int s=1; s<Ls; s++){
+        spProj5p(tmp, chi[ss+s-1]);
+        chi[ss+s] = psi[ss+s] - this->lee[s-1]*tmp;
+      }
+
+      // L_m^{-1}
+      for(int s=0; s<Ls-1; s++){ // Chi[ee] = 1 - sum[s<Ls-1] -leem[s]P_- chi
+        spProj5m(tmp, chi[ss+s]);
+        chi[ss+Ls-1] = chi[ss+Ls-1] - this->leem[s]*tmp;
+      }
+
+      // U_m^{-1} D^{-1}
+      for(int s=0; s<Ls-1; s++){ // Chi[s] + 1/d chi[s]
+        spProj5p(tmp, chi[ss+Ls-1]);
+        chi[ss+s] = (1.0/this->dee[s])*chi[ss+s] - (this->ueem[s]/this->dee[Ls-1])*tmp;
+      }
+      chi[ss+Ls-1] = (1.0/this->dee[Ls-1])*chi[ss+Ls-1];
+
+      // Apply U^{-1}
+      for(int s=Ls-2; s>=0; s--){
+        spProj5m(tmp, chi[ss+s+1]);
+        chi[ss+s] = chi[ss+s] - this->uee[s]*tmp;
+      }
+    }
+
+    this->MooeeInvTime += usecond();
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInv_shift(const FermionField &psi, FermionField &chi)
+  {
+    GridBase *grid = psi._grid;
+    int Ls = this->Ls;
+
+    chi.checkerboard = psi.checkerboard;
+
+    this->MooeeInvCalls++;
+    this->MooeeInvTime -= usecond();
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){
+
+      auto tmp1        = psi._odata[0];
+      auto tmp2        = psi._odata[0];
+      auto tmp2_spProj = psi._odata[0];
+
+      // Apply (L^{\prime})^{-1} and accumulate MooeeInv_shift_lc[j]*psi[j] in tmp2
+      chi[ss] = psi[ss]; // chi[0]=psi[0]
+      tmp2 = MooeeInv_shift_lc[0]*psi[ss];
+      for(int s=1; s<Ls; s++){
+        spProj5p(tmp1, chi[ss+s-1]);
+        chi[ss+s] = psi[ss+s] - this->lee[s-1]*tmp1;
+        tmp2 = tmp2 + MooeeInv_shift_lc[s]*psi[ss+s];
+      }
+      if(this->pm == 1){ spProj5p(tmp2_spProj, tmp2);}
+      else{ spProj5m(tmp2_spProj, tmp2); }
+
+      // L_m^{-1}
+      for(int s=0; s<Ls-1; s++){ // Chi[ee] = 1 - sum[s<Ls-1] -leem[s]P_- chi
+        spProj5m(tmp1, chi[ss+s]);
+        chi[ss+Ls-1] = chi[ss+Ls-1] - this->leem[s]*tmp1;
+      }
+
+      // U_m^{-1} D^{-1}
+      for(int s=0; s<Ls-1; s++){ // Chi[s] + 1/d chi[s]
+        spProj5p(tmp1, chi[ss+Ls-1]);
+        chi[ss+s] = (1.0/this->dee[s])*chi[ss+s] - (this->ueem[s]/this->dee[Ls-1])*tmp1;
+      }
+      // chi[ss+Ls-1] = (1.0/this->dee[Ls-1])*chi[ss+Ls-1] + MooeeInv_shift_norm[Ls-1]*tmp2_spProj;
+      chi[ss+Ls-1] = (1.0/this->dee[Ls-1])*chi[ss+Ls-1];
+      spProj5m(tmp1, chi[ss+Ls-1]);
+      chi[ss+Ls-1] = chi[ss+Ls-1] + MooeeInv_shift_norm[Ls-1]*tmp2_spProj;
+
+      // Apply U^{-1} and add shift term
+      for(int s=Ls-2; s>=0; s--){
+        chi[ss+s] = chi[ss+s] - this->uee[s]*tmp1;
+        spProj5m(tmp1, chi[ss+s]);
+        chi[ss+s] = chi[ss+s] + MooeeInv_shift_norm[s]*tmp2_spProj;
+      }
+    }
+
+    this->MooeeInvTime += usecond();
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInvDag(const FermionField &psi, FermionField &chi)
+  {
+    if(this->shift != 0.0){ MooeeInvDag_shift(psi,chi); return; }
+
+    GridBase *grid = psi._grid;
+    int Ls = this->Ls;
+
+    chi.checkerboard = psi.checkerboard;
+
+    this->MooeeInvCalls++;
+    this->MooeeInvTime -= usecond();
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){
+
+      auto tmp = psi._odata[0];
+
+      // Apply (U^{\prime})^{-dag}
+      chi[ss] = psi[ss];
+      for(int s=1; s<Ls; s++){
+        spProj5m(tmp, chi[ss+s-1]);
+        chi[ss+s] = psi[ss+s] - this->uee[s-1]*tmp;
+      }
+
+      // U_m^{-\dag}
+      for(int s=0; s<Ls-1; s++){
+        spProj5p(tmp, chi[ss+s]);
+        chi[ss+Ls-1] = chi[ss+Ls-1] - this->ueem[s]*tmp;
+      }
+
+      // L_m^{-\dag} D^{-dag}
+      for(int s=0; s<Ls-1; s++){
+        spProj5m(tmp, chi[ss+Ls-1]);
+        chi[ss+s] = (1.0/this->dee[s])*chi[ss+s] - (this->leem[s]/this->dee[Ls-1])*tmp;
+      }
+      chi[ss+Ls-1] = (1.0/this->dee[Ls-1])*chi[ss+Ls-1];
+
+      // Apply L^{-dag}
+      for(int s=Ls-2; s>=0; s--){
+        spProj5p(tmp, chi[ss+s+1]);
+        chi[ss+s] = chi[ss+s] - this->lee[s]*tmp;
+      }
+    }
+
+    this->MooeeInvTime += usecond();
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInvDag_shift(const FermionField &psi, FermionField &chi)
+  {
+    GridBase *grid = psi._grid;
+    int Ls = this->Ls;
+
+    chi.checkerboard = psi.checkerboard;
+
+    this->MooeeInvCalls++;
+    this->MooeeInvTime -= usecond();
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=Ls){
+
+      auto tmp1        = psi._odata[0];
+      auto tmp2        = psi._odata[0];
+      auto tmp2_spProj = psi._odata[0];
+
+      // Apply (U^{\prime})^{-dag} and accumulate MooeeInvDag_shift_lc[j]*psi[j] in tmp2
+      chi[ss] = psi[ss];
+      tmp2 = MooeeInvDag_shift_lc[0]*psi[ss];
+      for(int s=1; s<Ls; s++){
+        spProj5m(tmp1, chi[ss+s-1]);
+        chi[ss+s] = psi[ss+s] - this->uee[s-1]*tmp1;
+        tmp2 = tmp2 + MooeeInvDag_shift_lc[s]*psi[ss+s];
+      }
+      if(this->pm == 1){ spProj5p(tmp2_spProj, tmp2);}
+      else{ spProj5m(tmp2_spProj, tmp2); }
+
+      // U_m^{-\dag}
+      for(int s=0; s<Ls-1; s++){
+        spProj5p(tmp1, chi[ss+s]);
+        chi[ss+Ls-1] = chi[ss+Ls-1] - this->ueem[s]*tmp1;
+      }
+
+      // L_m^{-\dag} D^{-dag}
+      for(int s=0; s<Ls-1; s++){
+        spProj5m(tmp1, chi[ss+Ls-1]);
+        chi[ss+s] = (1.0/this->dee[s])*chi[ss+s] - (this->leem[s]/this->dee[Ls-1])*tmp1;
+      }
+      chi[ss+Ls-1] = (1.0/this->dee[Ls-1])*chi[ss+Ls-1];
+      spProj5p(tmp1, chi[ss+Ls-1]);
+      chi[ss+Ls-1] = chi[ss+Ls-1] + MooeeInvDag_shift_norm[Ls-1]*tmp2_spProj;
+
+      // Apply L^{-dag}
+      for(int s=Ls-2; s>=0; s--){
+        chi[ss+s] = chi[ss+s] - this->lee[s]*tmp1;
+        spProj5p(tmp1, chi[ss+s]);
+        chi[ss+s] = chi[ss+s] + MooeeInvDag_shift_norm[s]*tmp2_spProj;
+      }
+    }
+
+    this->MooeeInvTime += usecond();
+  }
+
+  #ifdef MOBIUS_EOFA_DPERP_CACHE
+
+  INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplF);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplD);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplF);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplD);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplF);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplD);
+
+  INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplFH);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplDF);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplFH);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplDF);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplFH);
+  INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplDF);
+
+  #endif
+
+}}

From bcefdd7c4eff147242ededf040653449c2d573c9 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sat, 19 Aug 2017 12:49:02 -0400
Subject: [PATCH 043/115] Align both allocator calls to 2MB

---
 lib/allocator/AlignedAllocator.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/allocator/AlignedAllocator.h b/lib/allocator/AlignedAllocator.h
index 7fd9496f..39734b53 100644
--- a/lib/allocator/AlignedAllocator.h
+++ b/lib/allocator/AlignedAllocator.h
@@ -186,9 +186,9 @@ public:
   pointer allocate(size_type __n, const void* _p= 0) 
   {
 #ifdef HAVE_MM_MALLOC_H
-    _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),128);
+    _Tp * ptr = (_Tp *) _mm_malloc(__n*sizeof(_Tp),GRID_ALLOC_ALIGN);
 #else
-    _Tp * ptr = (_Tp *) memalign(128,__n*sizeof(_Tp));
+    _Tp * ptr = (_Tp *) memalign(GRID_ALLOC_ALIGN,__n*sizeof(_Tp));
 #endif
     return ptr;
   }

From 9e658de2383620b5aa002f319b85442ab24d8115 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sat, 19 Aug 2017 12:52:44 -0400
Subject: [PATCH 044/115] Use Vector

---
 benchmarks/Benchmark_comms.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc
index 698f9d25..491fba1e 100644
--- a/benchmarks/Benchmark_comms.cc
+++ b/benchmarks/Benchmark_comms.cc
@@ -92,8 +92,8 @@ int main (int argc, char ** argv)
       RealD Nnode = Grid.NodeCount();
       RealD ppn = Nrank/Nnode;
 
-      std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
-      std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      Vector<Vector<HalfSpinColourVectorD> > xbuf(8,Vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      Vector<Vector<HalfSpinColourVectorD> > rbuf(8,Vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
@@ -172,8 +172,8 @@ int main (int argc, char ** argv)
       RealD Nnode = Grid.NodeCount();
       RealD ppn = Nrank/Nnode;
 
-      std::vector<std::vector<HalfSpinColourVectorD> > xbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
-      std::vector<std::vector<HalfSpinColourVectorD> > rbuf(8,std::vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      Vector<Vector<HalfSpinColourVectorD> > xbuf(8,Vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      Vector<Vector<HalfSpinColourVectorD> > rbuf(8,Vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
 
 
       int ncomm;

From d6472eda8d00c8d0ffc60760a4dd9462702ac00b Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sat, 19 Aug 2017 12:53:18 -0400
Subject: [PATCH 045/115] Use mmap

---
 lib/communicator/Communicator_base.cc | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 67bfaed0..6767495f 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -26,6 +26,10 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     *************************************************************************************/
     /*  END LEGAL */
 #include <Grid/GridCore.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <sys/mman.h>
 
 namespace Grid {
 
@@ -129,8 +133,15 @@ void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) {
   return NULL;
 }
 void CartesianCommunicator::ShmInitGeneric(void){
+#if 1
+  ShmCommBuf =(void *) mmap(NULL, MAX_MPI_SHM_BYTES, PROT_READ | PROT_WRITE,  MAP_HUGETLB| MAP_SHARED | MAP_ANONYMOUS, -1, 0); 
+  if (ShmCommBuf == (void *)MAP_FAILED) exit(EXIT_FAILURE);  
+  std::cout << "ShmCommBuf "<<ShmCommBuf<<std::endl;
+#else 
   ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES);
   ShmCommBuf=(void *)&ShmBufStorageVector[0];
+#endif
+  bzero(ShmCommBuf,MAX_MPI_SHM_BYTES);
 }
 
 #endif

From 2f619482b8160cd859da636ffbaad064ba4fc02f Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sat, 19 Aug 2017 12:53:59 -0400
Subject: [PATCH 046/115] Enable blocking stencil send

---
 lib/communicator/Communicator_mpit.cc | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc
index f522701c..c0fb47fd 100644
--- a/lib/communicator/Communicator_mpit.cc
+++ b/lib/communicator/Communicator_mpit.cc
@@ -242,20 +242,15 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 							 int recv_from_rank,
 							 int bytes,int dir)
 {
-  assert(false);
-  /*
   int myrank = _processor;
   int ierr;
   assert(dir < communicator_halo.size());
 
-  //  std::cout << " sending on communicator "<<dir<<" " <<communicator_halo[dir]<<std::endl;
-  // Give the CPU to MPI immediately; can use threads to overlap optionally
   ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,xmit_to_rank,myrank,
 		    recv,bytes,MPI_CHAR,recv_from_rank, recv_from_rank,
 		    communicator_halo[dir],MPI_STATUS_IGNORE);
   assert(ierr==0);
   return 2.0*bytes;
-  */
 }
 
 double CartesianCommunicator::StencilSendToRecvFrom(void *xmit,

From 0b0cf62193f6c44126bf0b01e77d6fcdcb0d09bd Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sat, 19 Aug 2017 13:18:50 -0400
Subject: [PATCH 047/115] Fix mpi 3 interface change

---
 lib/communicator/Communicator_mpi3.cc | 11 +++++++++++
 lib/communicator/Communicator_mpit.cc | 25 +++++++------------------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 46e4745c..e6e33d33 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -621,6 +621,17 @@ void CartesianCommunicator::SendToRecvFromBegin(std::vector<CommsRequest_t> &lis
   }
 }
 
+double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
+						     int dest,
+						     void *recv,
+						     int from,
+						     int bytes,int dir)
+{
+  std::vector<CommsRequest_t> list;
+  StencilSendToRecvFromBegin(list,xmit,dest,recv,from,bytes,dir);
+  StencilSendToRecvFromComplete(list,dir);
+}
+
 double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 							 void *xmit,
 							 int dest,
diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc
index c0fb47fd..9a9b26d2 100644
--- a/lib/communicator/Communicator_mpit.cc
+++ b/lib/communicator/Communicator_mpit.cc
@@ -242,17 +242,12 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 							 int recv_from_rank,
 							 int bytes,int dir)
 {
-  int myrank = _processor;
-  int ierr;
-  assert(dir < communicator_halo.size());
-
-  ierr=MPI_Sendrecv(xmit,bytes,MPI_CHAR,xmit_to_rank,myrank,
-		    recv,bytes,MPI_CHAR,recv_from_rank, recv_from_rank,
-		    communicator_halo[dir],MPI_STATUS_IGNORE);
-  assert(ierr==0);
-  return 2.0*bytes;
+  StencilSendToRecvFrom(xmit,xmit_to_rank,recv,recv_from_rank,bytes,dir);
 }
-
+void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
+{ 
+  // Do nothing
+};
 double CartesianCommunicator::StencilSendToRecvFrom(void *xmit,
 						    int xmit_to_rank,
 						    void *recv,
@@ -266,17 +261,11 @@ double CartesianCommunicator::StencilSendToRecvFrom(void *xmit,
   //  std::cout << " sending on communicator "<<dir<<" " <<communicator_halo[dir]<<std::endl;
   // Give the CPU to MPI immediately; can use threads to overlap optionally
   MPI_Request req[2];
-  MPI_Irecv(recv,bytes,MPI_CHAR,recv_from_rank,recv_from_rank,
-	    communicator_halo[dir],&req[1]);
-  MPI_Isend(xmit,bytes,MPI_CHAR,xmit_to_rank,myrank,
-	    communicator_halo[dir], &req[0]);
+  MPI_Irecv(recv,bytes,MPI_CHAR,recv_from_rank,recv_from_rank, communicator_halo[dir],&req[1]);
+  MPI_Isend(xmit,bytes,MPI_CHAR,xmit_to_rank,myrank,          communicator_halo[dir], &req[0]);
   MPI_Waitall(2, req, MPI_STATUSES_IGNORE);
   return 2.0*bytes;
 }
-void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
-{ 
-  // Do nothing
-};
 
 
 

From bfef525ed2474c0cfe1047e0351ab58ce525ff10 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sat, 19 Aug 2017 23:10:12 +0100
Subject: [PATCH 048/115] New benchmark prep

---
 benchmarks/Benchmark_ITT.cc | 518 ++++++++++++++++++++++++++++++++++++
 1 file changed, 518 insertions(+)
 create mode 100644 benchmarks/Benchmark_ITT.cc

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
new file mode 100644
index 00000000..4f16b1de
--- /dev/null
+++ b/benchmarks/Benchmark_ITT.cc
@@ -0,0 +1,518 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./benchmarks/Benchmark_memory_bandwidth.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+
+struct time_statistics{
+  double mean;
+  double err;
+  double min;
+  double max;
+
+  void statistics(std::vector<double> v){
+      double sum = std::accumulate(v.begin(), v.end(), 0.0);
+      mean = sum / v.size();
+
+      std::vector<double> diff(v.size());
+      std::transform(v.begin(), v.end(), diff.begin(), [=](double x) { return x - mean; });
+      double sq_sum = std::inner_product(diff.begin(), diff.end(), diff.begin(), 0.0);
+      err = std::sqrt(sq_sum / (v.size()*(v.size() - 1)));
+
+      auto result = std::minmax_element(v.begin(), v.end());
+      min = *result.first;
+      max = *result.second;
+}
+};
+
+void comms_header(){
+  std::cout <<GridLogMessage << " L  "<<"\t"<<" Ls  "<<"\t"
+            <<std::setw(11)<<"bytes"<<"MB/s uni (err/min/max)"<<"\t\t"<<"MB/s bidi (err/min/max)"<<std::endl;
+};
+
+Gamma::Algebra Gmu [] = {
+  Gamma::Algebra::GammaX,
+  Gamma::Algebra::GammaY,
+  Gamma::Algebra::GammaZ,
+  Gamma::Algebra::GammaT
+};
+struct controls {
+  int Opt;
+  int CommsOverlap;
+  Grid::CartesianCommunicator::CommunicatorPolicy_t CommsAsynch;
+  //  int HugePages;
+};
+
+class Benchmark {
+public:
+  static void Decomposition (void ) {
+
+    int threads = GridThread::GetThreads();
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "= Grid is setup to use "<<threads<<" threads"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage<<"Grid Default Decomposition patterns\n";
+    std::cout<<GridLogMessage<<"\tOpenMP threads : "<<GridThread::GetThreads()<<std::endl;
+    std::cout<<GridLogMessage<<"\tMPI tasks      : "<<GridCmdVectorIntToString(GridDefaultMpi())<<std::endl;
+    std::cout<<GridLogMessage<<"\tvReal          : "<<sizeof(vReal )*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vReal::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealF         : "<<sizeof(vRealF)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvRealD         : "<<sizeof(vRealD)*8    <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vRealD::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplex       : "<<sizeof(vComplex )*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplex::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexF      : "<<sizeof(vComplexF)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexF::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage<<"\tvComplexD      : "<<sizeof(vComplexD)*8 <<"bits ; " <<GridCmdVectorIntToString(GridDefaultSimd(4,vComplexD::Nsimd()))<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+  }
+
+  static void Comms(void)
+  {
+    int Nloop=100;
+    int nmu=0;
+    int maxlat=32;
+
+    std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplexD::Nsimd());
+    std::vector<int> mpi_layout  = GridDefaultMpi();
+
+    std::vector<double> t_time(Nloop);
+    time_statistics timestat;
+
+    std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "= Benchmarking threaded STENCIL halo exchange in "<<nmu<<" dimensions"<<std::endl;
+    std::cout<<GridLogMessage << "===================================================================================================="<<std::endl;
+    comms_header();
+
+    for(int lat=4;lat<=maxlat;lat+=4){
+      for(int Ls=8;Ls<=8;Ls*=2){
+
+	std::vector<int> latt_size  ({lat*mpi_layout[0],
+	      lat*mpi_layout[1],
+	      lat*mpi_layout[2],
+	      lat*mpi_layout[3]});
+
+	GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+	RealD Nrank = Grid._Nprocessors;
+	RealD Nnode = Grid.NodeCount();
+	RealD ppn = Nrank/Nnode;
+
+	std::vector<HalfSpinColourVectorD *> xbuf(8);
+	std::vector<HalfSpinColourVectorD *> rbuf(8);
+	Grid.ShmBufferFreeAll();
+	for(int d=0;d<8;d++){
+	  xbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	  rbuf[d] = (HalfSpinColourVectorD *)Grid.ShmBufferMalloc(lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	  bzero((void *)xbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	  bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
+	}
+
+	int ncomm;
+	int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+	double dbytes;
+	for(int i=0;i<Nloop;i++){
+	  double start=usecond();
+
+	  std::vector<CartesianCommunicator::CommsRequest_t> requests;
+	  dbytes=0;
+	  ncomm=0;
+
+	  parallel_for(int dir=0;dir<8;dir++){
+
+	    double tbytes;
+	    int mu =dir % 4;
+
+	    if (mpi_layout[mu]>1 ) {
+	        
+	      ncomm++;
+	      int xmit_to_rank;
+	      int recv_from_rank;
+	      if ( dir == mu ) { 
+		int comm_proc=1;
+		Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	      } else { 
+		int comm_proc = mpi_layout[mu]-1;
+		Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
+	      }
+#if 1
+	      tbytes= Grid.StencilSendToRecvFromBegin(requests,
+						      (void *)&xbuf[dir][0],
+						      xmit_to_rank,
+						      (void *)&rbuf[dir][0],
+						      recv_from_rank,
+						      bytes,dir);
+	      Grid.StencilSendToRecvFromComplete(requests,dir);
+#endif
+	      requests.resize(0);
+
+#pragma omp atomic
+	      dbytes+=tbytes;
+	    }
+	  }
+	  Grid.Barrier();
+	  double stop=usecond();
+	  t_time[i] = stop-start; // microseconds
+	}
+
+	timestat.statistics(t_time);
+
+	dbytes=dbytes*ppn;
+	double xbytes    = dbytes*0.5;
+	double rbytes    = dbytes*0.5;
+	double bidibytes = dbytes;
+
+
+	std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
+		 <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
+		 <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
+		 <<xbytes/timestat.max <<" "<< xbytes/timestat.min  
+		 << "\t\t"<<std::setw(7)<< bidibytes/timestat.mean<< "  " << bidibytes*timestat.err/(timestat.mean*timestat.mean) << " "
+		 << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
+
+ 
+      }
+    }    
+
+    return;
+  }
+
+  static void Memory(void)
+  {
+    const int Nvec=8;
+    typedef Lattice< iVector< vReal,Nvec> > LatticeVec;
+    typedef iVector<vReal,Nvec> Vec;
+
+    std::vector<int> simd_layout = GridDefaultSimd(Nd,vReal::Nsimd());
+    std::vector<int> mpi_layout  = GridDefaultMpi();
+
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "= Benchmarking a*x + y bandwidth"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
+    std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
+  
+  uint64_t lmax=48;
+#define NLOOP (10*lmax*lmax*lmax*lmax/lat/lat/lat/lat)
+
+    GridSerialRNG          sRNG;      sRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
+    for(int lat=8;lat<=lmax;lat+=4){
+
+      std::vector<int> latt_size  ({lat*mpi_layout[0],lat*mpi_layout[1],lat*mpi_layout[2],lat*mpi_layout[3]});
+      int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
+      GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
+
+      Vec rn ; random(sRNG,rn);
+
+      LatticeVec z(&Grid); z=rn;
+      LatticeVec x(&Grid); x=rn;
+      LatticeVec y(&Grid); y=rn;
+      double a=2.0;
+
+      uint64_t Nloop=NLOOP;
+
+      double start=usecond();
+      for(int i=0;i<Nloop;i++){
+	z=a*x-y;
+        x._odata[0]=z._odata[0]; // force serial dependency to prevent optimise away
+        y._odata[4]=z._odata[4];
+      }
+      double stop=usecond();
+      double time = (stop-start)/Nloop*1000;
+     
+      double flops=vol*Nvec*2;// mul,add
+      double bytes=3.0*vol*Nvec*sizeof(Real);
+      std::cout<<GridLogMessage<<std::setprecision(3) 
+	       << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.<<std::endl;
+
+    }
+  };
+
+
+  static void DWF(int Ls,int L)
+  {
+    RealD mass=0.1;
+    RealD M5  =1.8;
+
+    double mflops;
+    double mflops_best = 0;
+    double mflops_worst= 0;
+
+    ///////////////////////////////////////////////////////
+    // Set/Get the layout & grid size
+    ///////////////////////////////////////////////////////
+    int threads = GridThread::GetThreads();
+    std::vector<int> mpi = GridDefaultMpi(); assert(mpi.size()==4);
+    std::vector<int> local({L,L,L,L});
+
+    GridCartesian         * TmpGrid   = SpaceTimeGrid::makeFourDimGrid(std::vector<int>({64,64,64,64}), 
+								       GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    uint64_t NP = TmpGrid->RankCount();
+    uint64_t NN = TmpGrid->NodeCount();
+    uint64_t SHM=NP/NN;
+
+    std::vector<int> internal;
+    if      ( SHM == 1 )   internal = std::vector<int>({1,1,1,1});
+    else if ( SHM == 2 )   internal = std::vector<int>({2,1,1,1});
+    else if ( SHM == 4 )   internal = std::vector<int>({2,2,1,1});
+    else if ( SHM == 8 )   internal = std::vector<int>({2,2,2,1});
+    else assert(0);
+
+    std::vector<int> nodes({mpi[0]/internal[0],mpi[1]/internal[1],mpi[2]/internal[2],mpi[3]/internal[3]});
+    std::vector<int> latt4({local[0]*nodes[0],local[1]*nodes[1],local[2]*nodes[2],local[3]*nodes[3]});
+
+    ///////// Welcome message ////////////
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "Benchmark DWF on "<<L<<"^4 local volume "<<std::endl;
+    std::cout<<GridLogMessage << "* Global volume  : "<<GridCmdVectorIntToString(latt4)<<std::endl;
+    std::cout<<GridLogMessage << "* Ls             : "<<Ls<<std::endl;
+    std::cout<<GridLogMessage << "* MPI ranks      : "<<GridCmdVectorIntToString(mpi)<<std::endl;
+    std::cout<<GridLogMessage << "* Intranode      : "<<GridCmdVectorIntToString(internal)<<std::endl;
+    std::cout<<GridLogMessage << "* nodes          : "<<GridCmdVectorIntToString(nodes)<<std::endl;
+    std::cout<<GridLogMessage << "* Using "<<threads<<" threads"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+
+    ///////// Lattice Init ////////////
+    GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+    GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+    GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+    
+    ///////// RNG Init ////////////
+    std::vector<int> seeds4({1,2,3,4});
+    std::vector<int> seeds5({5,6,7,8});
+    GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+    GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+    std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
+
+    ///////// Source preparation ////////////
+    LatticeFermion src   (FGrid); random(RNG5,src);
+    LatticeFermion ref   (FGrid);
+    LatticeFermion tmp   (FGrid);
+
+    RealD N2 = 1.0/::sqrt(norm2(src));
+    src = src*N2;
+    
+    LatticeGaugeField Umu(UGrid);  SU3::HotConfiguration(RNG4,Umu); 
+
+    DomainWallFermionR Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+
+    ////////////////////////////////////
+    // Naive wilson implementation
+    ////////////////////////////////////
+    {
+      LatticeGaugeField Umu5d(FGrid); 
+      std::vector<LatticeColourMatrix> U(4,FGrid);
+      for(int ss=0;ss<Umu._grid->oSites();ss++){
+	for(int s=0;s<Ls;s++){
+	  Umu5d._odata[Ls*ss+s] = Umu._odata[ss];
+	}
+      }
+      ref = zero;
+      for(int mu=0;mu<Nd;mu++){
+	U[mu] = PeekIndex<LorentzIndex>(Umu5d,mu);
+      }
+      for(int mu=0;mu<Nd;mu++){
+	
+	tmp = U[mu]*Cshift(src,mu+1,1);
+	ref=ref + tmp - Gamma(Gmu[mu])*tmp;
+	
+	tmp =adj(U[mu])*src;
+	tmp =Cshift(tmp,mu+1,-1);
+	ref=ref + tmp + Gamma(Gmu[mu])*tmp;
+      }
+      ref = -0.5*ref;
+    }
+
+    LatticeFermion src_e (FrbGrid);
+    LatticeFermion src_o (FrbGrid);
+    LatticeFermion r_e   (FrbGrid);
+    LatticeFermion r_o   (FrbGrid);
+    LatticeFermion r_eo  (FGrid);
+    LatticeFermion err   (FGrid);
+    {
+
+      pickCheckerboard(Even,src_e,src);
+      pickCheckerboard(Odd,src_o,src);
+
+#if defined(AVX512) 
+      const int num_cases = 6;
+#else
+      const int num_cases = 4;
+#endif
+      controls Cases [] = {
+#if defined(AVX512) 
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+#endif
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  }
+      }; 
+
+      for(int c=0;c<num_cases;c++) {
+
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+	if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+	if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+
+	QCD::WilsonKernelsStatic::Comms = Cases[c].CommsOverlap;
+	QCD::WilsonKernelsStatic::Opt   = Cases[c].Opt;
+	CartesianCommunicator::SetCommunicatorPolicy(Cases[c].CommsAsynch);
+	int nwarm = 10;
+	double t0=usecond();
+	FGrid->Barrier();
+	for(int i=0;i<nwarm;i++){
+	  Dw.DhopEO(src_o,r_e,DaggerNo);
+	}
+	FGrid->Barrier();
+	double t1=usecond();
+	uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0);
+	FGrid->Broadcast(0,&ncall,sizeof(ncall));
+
+	//	std::cout << GridLogMessage << " Estimate " << ncall << " calls per second"<<std::endl;
+	Dw.ZeroCounters();
+
+	time_statistics timestat;
+	std::vector<double> t_time(ncall);
+	for(uint64_t i=0;i<ncall;i++){
+	  t0=usecond();
+	  Dw.DhopEO(src_o,r_e,DaggerNo);
+	  t1=usecond();
+	  t_time[i] = t1-t0;
+	}
+	FGrid->Barrier();
+	
+	double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+	double flops=(1344.0*volume)/2;
+	double mf_hi, mf_lo, mf_err;
+
+	timestat.statistics(t_time);
+	mf_hi = flops/timestat.min;
+	mf_lo = flops/timestat.max;
+	mf_err= flops/timestat.min * timestat.err/timestat.mean;
+
+	mflops = flops/timestat.mean;
+
+	if ( mflops_best == 0   ) mflops_best = mflops;
+	if ( mflops_worst== 0   ) mflops_worst= mflops;
+	if ( mflops>mflops_best ) mflops_best = mflops;
+	if ( mflops<mflops_worst) mflops_worst= mflops;
+
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"Deo mflop/s =   "<< mflops << " ("<<mf_err<<") " << mf_lo<<"-"<<mf_hi <<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"Deo mflop/s per rank   "<< mflops/NP<<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"Deo mflop/s per node   "<< mflops/NN<<std::endl;
+
+	Dw.Report();
+
+	Dw.DhopEO(src_o,r_e,DaggerNo);
+	Dw.DhopOE(src_e,r_o,DaggerNo);
+	setCheckerboard(r_eo,r_o);
+	setCheckerboard(r_eo,r_e);
+	err = r_eo-ref; 
+	std::cout<<GridLogMessage << "norm diff   "<< norm2(err)<<std::endl;
+	assert((norm2(err)<1.0e-4));
+
+      }
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Best  mflop/s        =   "<< mflops_best <<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Worst mflop/s        =   "<< mflops_worst<<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< mflops_worst/mflops_best <<std::endl;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+    }
+  }
+
+};
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  CartesianCommunicator::SetCommunicatorPolicy(CartesianCommunicator::CommunicatorPolicySequential);
+  LebesgueOrder::Block = std::vector<int>({2,2,2,2});
+
+  Benchmark::Decomposition();
+
+  int do_memory=1;
+  int do_comms =1;
+  int do_su3   =0;
+  int do_wilson=1;
+  int do_dwf   =1;
+
+  if ( do_memory ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Memory benchmark " <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::Memory();
+  }
+
+  if ( do_comms ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Communications benchmark " <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::Comms();
+  }
+
+  if ( do_su3 ) {
+    // empty for now
+  }
+
+  if ( do_wilson ) {
+    int Ls=1;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Wilson dslash 4D vectorised" <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::DWF(Ls,16);
+    Benchmark::DWF(Ls,24);
+    Benchmark::DWF(Ls,32);
+  }
+
+  if ( do_dwf ) {
+    int Ls=16;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Domain wall dslash 4D vectorised" <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::DWF(Ls,8);
+    Benchmark::DWF(Ls,12);
+    Benchmark::DWF(Ls,16);
+    Benchmark::DWF(Ls,24);
+  }
+
+  Grid_finalize();
+}

From 6d0d064a6c62f2c269df1a6aa557152543f8c186 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sat, 19 Aug 2017 23:11:30 +0100
Subject: [PATCH 049/115] Update TODO

---
 TODO | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/TODO b/TODO
index 001c6c0c..cccc5f45 100644
--- a/TODO
+++ b/TODO
@@ -2,18 +2,18 @@ TODO:
 ---------------
 
 Large item work list:
-1)- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O
 
+1)- BG/Q port and check
 2)- Christoph's local basis expansion Lanczos
-3)- BG/Q port and check
-4)- Precision conversion and sort out localConvert      <-- partial
+3)- Precision conversion and sort out localConvert      <-- partial
   - Consistent linear solver flop count/rate -- PARTIAL, time but no flop/s yet
-5)- Physical propagator interface
-6)- Conserved currents
-7)- Multigrid Wilson and DWF, compare to other Multigrid implementations
-8)- HDCR resume
+4)- Physical propagator interface
+5)- Conserved currents
+6)- Multigrid Wilson and DWF, compare to other Multigrid implementations
+7)- HDCR resume
 
 Recent DONE 
+-- MultiRHS with spread out extra dim -- Go through filesystem with SciDAC I/O
 -- Lanczos Remove DenseVector, DenseMatrix; Use Eigen instead. <-- DONE
 -- GaugeFix into central location                      <-- DONE
 -- Scidac and Ildg metadata handling                   <-- DONE

From a446d95c3393d697f987434ac594950d18017b7a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 20 Aug 2017 01:10:50 +0100
Subject: [PATCH 050/115] Trying to pass TeamCity and Travis

---
 benchmarks/Benchmark_ITT.cc               | 12 ++++++------
 lib/communicator/Communicator_base.cc     |  6 +++++-
 lib/communicator/Communicator_base.h      | 19 +++++++++++++------
 lib/communicator/Communicator_mpi3.cc     | 17 +++++++++++++----
 lib/qcd/action/fermion/WilsonFermion5D.cc | 18 ++++++++++--------
 lib/stencil/Stencil.h                     |  7 ++++++-
 lib/util/Init.cc                          | 18 ++++++++++++++----
 7 files changed, 67 insertions(+), 30 deletions(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 4f16b1de..9bf7d0a5 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -218,7 +218,7 @@ public:
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
   
   uint64_t lmax=48;
-#define NLOOP (10*lmax*lmax*lmax*lmax/lat/lat/lat/lat)
+#define NLOOP (50*lmax*lmax*lmax*lmax/lat/lat/lat/lat)
 
     GridSerialRNG          sRNG;      sRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
     for(int lat=8;lat<=lmax;lat+=4){
@@ -368,7 +368,7 @@ public:
       const int num_cases = 4;
 #endif
       controls Cases [] = {
-#if defined(AVX512) 
+#ifdef AVX512
 	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
 	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
 #endif
@@ -380,6 +380,10 @@ public:
 
       for(int c=0;c<num_cases;c++) {
 
+	QCD::WilsonKernelsStatic::Comms = Cases[c].CommsOverlap;
+	QCD::WilsonKernelsStatic::Opt   = Cases[c].Opt;
+	CartesianCommunicator::SetCommunicatorPolicy(Cases[c].CommsAsynch);
+
 	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
 	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
@@ -390,10 +394,6 @@ public:
 	if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
 	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
-
-	QCD::WilsonKernelsStatic::Comms = Cases[c].CommsOverlap;
-	QCD::WilsonKernelsStatic::Opt   = Cases[c].Opt;
-	CartesianCommunicator::SetCommunicatorPolicy(Cases[c].CommsAsynch);
 	int nwarm = 10;
 	double t0=usecond();
 	FGrid->Barrier();
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 6767495f..3ce3a774 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -41,6 +41,7 @@ uint64_t            CartesianCommunicator::MAX_MPI_SHM_BYTES   = 128*1024*1024;
 CartesianCommunicator::CommunicatorPolicy_t  
 CartesianCommunicator::CommunicatorPolicy= CartesianCommunicator::CommunicatorPolicyConcurrent;
 int CartesianCommunicator::nCommThreads = -1;
+int CartesianCommunicator::Hugepages = 0;
 
 /////////////////////////////////
 // Alloc, free shmem region
@@ -134,7 +135,10 @@ void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) {
 }
 void CartesianCommunicator::ShmInitGeneric(void){
 #if 1
-  ShmCommBuf =(void *) mmap(NULL, MAX_MPI_SHM_BYTES, PROT_READ | PROT_WRITE,  MAP_HUGETLB| MAP_SHARED | MAP_ANONYMOUS, -1, 0); 
+
+  int mmap_flag = MAP_SHARED | MAP_ANONYMOUS;
+  if ( Hugepages ) mmap_flag |= MAP_HUGETLB;
+  ShmCommBuf =(void *) mmap(NULL, MAX_MPI_SHM_BYTES, PROT_READ | PROT_WRITE, mmap_flag, -1, 0); 
   if (ShmCommBuf == (void *)MAP_FAILED) exit(EXIT_FAILURE);  
   std::cout << "ShmCommBuf "<<ShmCommBuf<<std::endl;
 #else 
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index 84dbedb4..ac7d94f3 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -50,13 +50,24 @@ namespace Grid {
 class CartesianCommunicator {
   public:    
 
-  // 65536 ranks per node adequate for now
+
+  ////////////////////////////////////////////
+  // Isend/Irecv/Wait, or Sendrecv blocking
+  ////////////////////////////////////////////
+  enum CommunicatorPolicy_t { CommunicatorPolicyConcurrent, CommunicatorPolicySequential };
+  static CommunicatorPolicy_t CommunicatorPolicy;
+  static void SetCommunicatorPolicy(CommunicatorPolicy_t policy ) { CommunicatorPolicy = policy; }
+
+  ///////////////////////////////////////////
+  // Up to 65536 ranks per node adequate for now
   // 128MB shared memory for comms enought for 48^4 local vol comms
   // Give external control (command line override?) of this
-
+  ///////////////////////////////////////////
   static const int MAXLOG2RANKSPERNODE = 16;            
   static uint64_t  MAX_MPI_SHM_BYTES;
   static int       nCommThreads;
+  // use explicit huge pages
+  static int       Hugepages;
 
   // Communicator should know nothing of the physics grid, only processor grid.
   int              _Nprocessors;     // How many in all
@@ -122,10 +133,6 @@ class CartesianCommunicator {
   /////////////////////////////////
   static void * ShmCommBuf;
 
-  // Isend/Irecv/Wait, or Sendrecv blocking
-  enum CommunicatorPolicy_t { CommunicatorPolicyConcurrent, CommunicatorPolicySequential };
-  static CommunicatorPolicy_t CommunicatorPolicy;
-  static void SetCommunicatorPolicy(CommunicatorPolicy_t policy ) { CommunicatorPolicy = policy; }
   
   size_t heap_top;
   size_t heap_bytes;
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index e6e33d33..4f769971 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -41,8 +41,13 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #ifdef HAVE_NUMAIF_H
 #include <numaif.h>
 #endif
+
+// Make up for linex deficiencies
 #ifndef SHM_HUGETLB
-#define SHM_HUGETLB 04000
+#define SHM_HUGETLB 0x0
+#endif
+#ifndef MAP_HUGETLB
+#define MAP_HUGETLB 0x0
 #endif
 
 namespace Grid {
@@ -213,8 +218,11 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
       int fd=shm_open(shm_name,O_RDWR|O_CREAT,0666);
       if ( fd < 0 ) {	perror("failed shm_open");	assert(0);      }
       ftruncate(fd, size);
+      
+      int mmap_flag = MAP_SHARED;
+      if (Hugepages) mmap_flag |= MAP_HUGETLB;
+      void * ptr =  mmap(NULL,size, PROT_READ | PROT_WRITE, mmap_flag, fd, 0);
 
-      void * ptr =  mmap(NULL,size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
       if ( ptr == MAP_FAILED ) {       perror("failed mmap");      assert(0);    }
       assert(((uint64_t)ptr&0x3F)==0);
 
@@ -628,8 +636,9 @@ double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
 						     int bytes,int dir)
 {
   std::vector<CommsRequest_t> list;
-  StencilSendToRecvFromBegin(list,xmit,dest,recv,from,bytes,dir);
+  double offbytes = StencilSendToRecvFromBegin(list,xmit,dest,recv,from,bytes,dir);
   StencilSendToRecvFromComplete(list,dir);
+  return offbytes;
 }
 
 double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
@@ -671,7 +680,7 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
   }
 
   if ( CommunicatorPolicy == CommunicatorPolicySequential ) { 
-    this->StencilSendToRecvFromComplete(list);
+    this->StencilSendToRecvFromComplete(list,dir);
   }
 
   return off_node_bytes;
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index 0b6c9e3d..404ecce0 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -135,10 +135,11 @@ WilsonFermion5D<Impl>::WilsonFermion5D(GaugeField &_Umu,
 template<class Impl>
 void WilsonFermion5D<Impl>::Report(void)
 {
-    std::vector<int> latt = GridDefaultLatt();          
-    RealD volume = Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt[mu];
-    RealD NP = _FourDimGrid->_Nprocessors;
-    RealD NN = _FourDimGrid->NodeCount();
+  RealD NP     = _FourDimGrid->_Nprocessors;
+  RealD NN     = _FourDimGrid->NodeCount();
+  RealD volume = Ls;  
+  std::vector<int> latt = _FourDimGrid->GlobalDimensions();
+  for(int mu=0;mu<Nd;mu++) volume=volume*latt[mu];
 
   if ( DhopCalls > 0 ) {
     std::cout << GridLogMessage << "#### Dhop calls report " << std::endl;
@@ -390,17 +391,18 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
   st.CommsMergeSHM(compressor);// Could do this inside parallel region overlapped with comms
   DhopFaceTime+=usecond();
 
-  // Rely on async comms; start comms before merge of local data
   double ctime=0;
   double ptime=0;
-  //  DhopComputeTime-=usecond();
-  //  DhopCommTime-=usecond();
+
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Ugly explicit thread mapping introduced for OPA reasons.
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
 #pragma omp parallel reduction(max:ctime) reduction(max:ptime)
   { 
     int tid = omp_get_thread_num();
     int nthreads = omp_get_num_threads();
     int ncomms = CartesianCommunicator::nCommThreads;
-    if (ncomms == -1) ncomms = st.Packets.size(); 
+    if (ncomms == -1) ncomms = 1;
     assert(nthreads > ncomms);
     if (tid >= ncomms) {
       double start = usecond();
diff --git a/lib/stencil/Stencil.h b/lib/stencil/Stencil.h
index d1d7a7e0..cca67587 100644
--- a/lib/stencil/Stencil.h
+++ b/lib/stencil/Stencil.h
@@ -252,10 +252,15 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
   //////////////////////////////////////////
   void CommunicateThreaded()
   {
+#ifdef GRID_OMP
     // must be called in parallel region
     int mythread = omp_get_thread_num();
     int nthreads = CartesianCommunicator::nCommThreads;
-    if (nthreads == -1) nthreads = Packets.size();
+#else
+    int mythread = 0;
+    int nthreads = 1;
+#endif
+    if (nthreads == -1) nthreads = 1;
     if (mythread < nthreads) {
       for (int i = mythread; i < Packets.size(); i += nthreads) {
 	double start = usecond();
diff --git a/lib/util/Init.cc b/lib/util/Init.cc
index 39a726cf..3fd8b4cd 100644
--- a/lib/util/Init.cc
+++ b/lib/util/Init.cc
@@ -222,6 +222,11 @@ void Grid_init(int *argc,char ***argv)
     CartesianCommunicator::MAX_MPI_SHM_BYTES = MB*1024*1024;
   }
 
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--shm-hugepages") ){
+    CartesianCommunicator::Hugepages = 1;
+  }
+
+
   if( GridCmdOptionExists(*argv,*argv+*argc,"--debug-signals") ){
     Grid_debug_handler_init();
   }
@@ -304,6 +309,7 @@ void Grid_init(int *argc,char ***argv)
     std::cout<<GridLogMessage<<"  --threads n     : default number of OMP threads"<<std::endl;
     std::cout<<GridLogMessage<<"  --grid n.n.n.n  : default Grid size"<<std::endl;
     std::cout<<GridLogMessage<<"  --shm  M        : allocate M megabytes of shared memory for comms"<<std::endl;
+    std::cout<<GridLogMessage<<"  --shm-hugepages : use explicit huge pages in mmap call "<<std::endl;    
     std::cout<<GridLogMessage<<std::endl;
     std::cout<<GridLogMessage<<"Verbose and debug:"<<std::endl;
     std::cout<<GridLogMessage<<std::endl;
@@ -317,7 +323,7 @@ void Grid_init(int *argc,char ***argv)
     std::cout<<GridLogMessage<<std::endl;
     std::cout<<GridLogMessage<<"  --comms-concurrent : Asynchronous MPI calls; several dirs at a time "<<std::endl;    
     std::cout<<GridLogMessage<<"  --comms-sequential : Synchronous MPI calls; one dirs at a time "<<std::endl;    
-    std::cout<<GridLogMessage<<"  --comms-overlap : Overlap comms with compute "<<std::endl;    
+    std::cout<<GridLogMessage<<"  --comms-overlap    : Overlap comms with compute "<<std::endl;    
     std::cout<<GridLogMessage<<std::endl;
     std::cout<<GridLogMessage<<"  --dslash-generic: Wilson kernel for generic Nc"<<std::endl;    
     std::cout<<GridLogMessage<<"  --dslash-unroll : Wilson kernel for Nc=3"<<std::endl;    
@@ -356,12 +362,13 @@ void Grid_init(int *argc,char ***argv)
   if( GridCmdOptionExists(*argv,*argv+*argc,"--comms-sequential") ){
     CartesianCommunicator::SetCommunicatorPolicy(CartesianCommunicator::CommunicatorPolicySequential);
   }
+
   if( GridCmdOptionExists(*argv,*argv+*argc,"--lebesgue") ){
     LebesgueOrder::UseLebesgueOrder=1;
   }
   CartesianCommunicator::nCommThreads = -1;
-  if( GridCmdOptionExists(*argv,*argv+*argc,"--commthreads") ){
-    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--commthreads");
+  if( GridCmdOptionExists(*argv,*argv+*argc,"--comms-threads") ){
+    arg= GridCmdOptionPayload(*argv,*argv+*argc,"--comms-threads");
     GridCmdOptionInt(arg,CartesianCommunicator::nCommThreads);
   }
   if( GridCmdOptionExists(*argv,*argv+*argc,"--cacheblocking") ){
@@ -378,7 +385,10 @@ void Grid_init(int *argc,char ***argv)
 		  Grid_default_latt,
 		  Grid_default_mpi);
 
-  std::cout << GridLogDebug << "Requesting "<< CartesianCommunicator::MAX_MPI_SHM_BYTES <<" byte stencil comms buffers "<<std::endl;
+  std::cout << GridLogMessage << "Requesting "<< CartesianCommunicator::MAX_MPI_SHM_BYTES <<" byte stencil comms buffers "<<std::endl;
+  if ( CartesianCommunicator::Hugepages) {
+    std::cout << GridLogMessage << "Mapped stencil comms buffers as MAP_HUGETLB "<<std::endl;
+  }
 
   if( GridCmdOptionExists(*argv,*argv+*argc,"--decomposition") ){
     std::cout<<GridLogMessage<<"Grid Default Decomposition patterns\n";

From 383ca7d39239d99a90e11df7a73918bea0268927 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 20 Aug 2017 01:27:48 +0100
Subject: [PATCH 051/115] Switch off comms for now until
 feature/multi-communicator is merged

---
 benchmarks/Benchmark_ITT.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 4f16b1de..91524149 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -160,7 +160,7 @@ public:
 		int comm_proc = mpi_layout[mu]-1;
 		Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	      }
-#if 1
+#if 0
 	      tbytes= Grid.StencilSendToRecvFromBegin(requests,
 						      (void *)&xbuf[dir][0],
 						      xmit_to_rank,

From 11062fb6861153ffafa6d821f8ee53f01f5f72a4 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 20 Aug 2017 01:37:07 +0100
Subject: [PATCH 052/115] Comms none fail fix

---
 lib/communicator/Communicator_base.cc | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 3ce3a774..2e6626be 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -102,6 +102,18 @@ int                      CartesianCommunicator::NodeCount(void)    { return Proc
 int                      CartesianCommunicator::RankCount(void)    { return ProcessorCount();};
 #endif
 #if !defined( GRID_COMMS_MPI3) && !defined (GRID_COMMS_MPIT)
+double CartesianCommunicator::StencilSendToRecvFrom( void *xmit,
+						     int xmit_to_rank,
+						     void *recv,
+						     int recv_from_rank,
+						     int bytes, int dir)
+{
+  std::vector<CommsRequest_t> list;
+  // Discard the "dir"
+  SendToRecvFromBegin   (list,xmit,xmit_to_rank,recv,recv_from_rank,bytes);
+  SendToRecvFromComplete(list);
+  return 2.0*bytes;
+}
 double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsRequest_t> &list,
 							 void *xmit,
 							 int xmit_to_rank,

From 1cdf99966810227f180452393973c87ae4a301c4 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 20 Aug 2017 02:39:10 +0100
Subject: [PATCH 053/115] Moving multicommunicator into mpi3 also for threading

---
 lib/communicator/Communicator_base.h  |  8 ++++----
 lib/communicator/Communicator_mpi3.cc | 12 ++++++++++--
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index ac7d94f3..ac866ced 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -78,15 +78,15 @@ class CartesianCommunicator {
 
 #if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPI3) || defined (GRID_COMMS_MPIT)
   static MPI_Comm communicator_world;
-         MPI_Comm communicator;
+
+  MPI_Comm              communicator;
+  std::vector<MPI_Comm> communicator_halo;
+
   typedef MPI_Request CommsRequest_t;
 #else 
   typedef int CommsRequest_t;
 #endif
 
-#if defined (GRID_COMMS_MPIT)
-  std::vector<MPI_Comm> communicator_halo;
-#endif
 
   ////////////////////////////////////////////////////////////////////
   // Helper functionality for SHM Windows common to all other impls
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 4f769971..9e5dfb97 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -405,8 +405,14 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
 { 
   int ierr;
   communicator=communicator_world;
+
   _ndimension = processors.size();
 
+  communicator_halo.resize (2*_ndimension);
+  for(int i=0;i<_ndimension*2;i++){
+    MPI_Comm_dup(communicator,&communicator_halo[i]);
+  }
+
   ////////////////////////////////////////////////////////////////
   // Assert power of two shm_size.
   ////////////////////////////////////////////////////////////////
@@ -648,6 +654,8 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 							 int from,
 							 int bytes,int dir)
 {
+  assert(dir < communicator_halo.size());
+
   MPI_Request xrq;
   MPI_Request rrq;
 
@@ -666,14 +674,14 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
   gfrom = MPI_UNDEFINED;
 #endif
   if ( gfrom ==MPI_UNDEFINED) {
-    ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator,&rrq);
+    ierr=MPI_Irecv(recv, bytes, MPI_CHAR,from,from,communicator_halo[dir],&rrq);
     assert(ierr==0);
     list.push_back(rrq);
     off_node_bytes+=bytes;
   }
 
   if ( gdest == MPI_UNDEFINED ) {
-    ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator,&xrq);
+    ierr =MPI_Isend(xmit, bytes, MPI_CHAR,dest,_processor,communicator_halo[dir],&xrq);
     assert(ierr==0);
     list.push_back(xrq);
     off_node_bytes+=bytes;

From ae56e556c64fe1b2cdd2d302ef63294cc322aeaf Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 20 Aug 2017 02:53:12 +0100
Subject: [PATCH 054/115] finalise issue on new OPA revert

---
 benchmarks/Benchmark_dwf.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index 98ce0a07..3858226e 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -503,9 +503,9 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "norm diff even  "<< norm2(src_e)<<std::endl;
   std::cout<<GridLogMessage << "norm diff odd   "<< norm2(src_o)<<std::endl;
 
-  //assert(norm2(src_e)<1.0e-4);
-  //assert(norm2(src_o)<1.0e-4);
-  exit(0);
+  assert(norm2(src_e)<1.0e-4);
+  assert(norm2(src_o)<1.0e-4);
   Grid_finalize();
+  exit(0);
 }
 

From b49bec0cec4a876462fa1f042556d595f3d88416 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 20 Aug 2017 03:08:54 +0100
Subject: [PATCH 055/115] MAP_HUGETLB portability fix

---
 lib/communicator/Communicator_base.cc |  2 ++
 lib/communicator/Communicator_mpi3.cc | 15 +++++++--------
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 2e6626be..3378c56a 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -149,7 +149,9 @@ void CartesianCommunicator::ShmInitGeneric(void){
 #if 1
 
   int mmap_flag = MAP_SHARED | MAP_ANONYMOUS;
+#ifdef MAP_HUGETLB
   if ( Hugepages ) mmap_flag |= MAP_HUGETLB;
+#endif
   ShmCommBuf =(void *) mmap(NULL, MAX_MPI_SHM_BYTES, PROT_READ | PROT_WRITE, mmap_flag, -1, 0); 
   if (ShmCommBuf == (void *)MAP_FAILED) exit(EXIT_FAILURE);  
   std::cout << "ShmCommBuf "<<ShmCommBuf<<std::endl;
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 9e5dfb97..204993fd 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -42,13 +42,6 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 #include <numaif.h>
 #endif
 
-// Make up for linex deficiencies
-#ifndef SHM_HUGETLB
-#define SHM_HUGETLB 0x0
-#endif
-#ifndef MAP_HUGETLB
-#define MAP_HUGETLB 0x0
-#endif
 
 namespace Grid {
 
@@ -220,7 +213,9 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
       ftruncate(fd, size);
       
       int mmap_flag = MAP_SHARED;
+#ifdef MAP_HUGETLB
       if (Hugepages) mmap_flag |= MAP_HUGETLB;
+#endif
       void * ptr =  mmap(NULL,size, PROT_READ | PROT_WRITE, mmap_flag, fd, 0);
 
       if ( ptr == MAP_FAILED ) {       perror("failed mmap");      assert(0);    }
@@ -274,7 +269,11 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
     for(int r=0;r<ShmSize;r++){
       size_t size = CartesianCommunicator::MAX_MPI_SHM_BYTES;
       key_t key   = 0x4545 + r;
-      if ((shmids[r]= shmget(key,size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W)) < 0) {
+      int flags = IPC_CREAT | SHM_R | SHM_W;
+#ifdef SHM_HUGETLB
+      flags|=SHM_HUGETLB;
+#endif
+      if ((shmids[r]= shmget(key,size, flags)) < 0) {
 	int errsv = errno;
 	printf("Errno %d\n",errsv);
 	perror("shmget");

From ab50145001d7a2f27c9f724440e0bfd88a6373e2 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Tue, 22 Aug 2017 17:12:25 -0400
Subject: [PATCH 056/115] Implemented first, unoptimized version of
 hand-unrolled G-parity kernels Improved Test_gparity

---
 lib/qcd/action/fermion/FermionOperatorImpl.h |  81 +++-
 lib/qcd/action/fermion/WilsonKernelsHand.cc  | 456 +++++++++++++------
 tests/core/Test_gparity.cc                   | 148 +++---
 3 files changed, 485 insertions(+), 200 deletions(-)

diff --git a/lib/qcd/action/fermion/FermionOperatorImpl.h b/lib/qcd/action/fermion/FermionOperatorImpl.h
index 524179f5..1e344521 100644
--- a/lib/qcd/action/fermion/FermionOperatorImpl.h
+++ b/lib/qcd/action/fermion/FermionOperatorImpl.h
@@ -425,6 +425,22 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
     ////////////////////////////////////////////////////////////////////////////////////////
     // Flavour doubled spinors; is Gparity the only? what about C*?
     ////////////////////////////////////////////////////////////////////////////////////////
+namespace GparityWilsonImpl_helper{
+  template<typename Get, typename A,typename B>
+  struct getAB;
+  
+  template<typename A,typename B>
+  struct getAB<A,A,B>{
+    static inline A & ref(A &a, B &b){ return a; }
+  };
+  template<typename A,typename B>
+  struct getAB<B,A,B>{
+    static inline B & ref(A &a, B &b){ return b; }
+  };
+};
+  
+
+  
 template <class S, int Nrepresentation, class Options=CoeffReal>
 class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresentation> > {
  public:
@@ -462,7 +478,10 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
       
  ImplParams Params;
 
- GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
+ std::vector<SiteSpinor, alignedAllocator<SiteSpinor> > tmp_full;
+ std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > tmp_half;
+
+ GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p), tmp_full(GridThread::GetThreads()), tmp_half(GridThread::GetThreads()){};
 
  bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
 
@@ -538,6 +557,66 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
    
  }
 
+
+ template <class ref>
+ inline void loadLinkElement(Simd &reg, ref &memory) {
+   reg = memory;
+ }
+
+ template<typename SiteSpinorType>
+ void GparityTwistPermute(SiteSpinorType &into,  const SiteSpinorType &from, const int direction, const int distance, const int perm, GridBase* grid){
+   typedef typename SiteSpinorType::scalar_object sobj;
+   sobj stmp;
+   std::vector<sobj> vals(grid->Nsimd());
+   extract(from,vals);
+   std::vector<int> icoor;
+   for(int s=0;s<grid->Nsimd();s++){
+     grid->iCoorFromIindex(icoor,s);
+     assert((icoor[direction]==0)||(icoor[direction]==1));
+
+     int permute_lane;
+     if ( distance == 1) {
+       permute_lane = icoor[direction]?1:0;
+     } else {
+       permute_lane = icoor[direction]?0:1;
+     }
+     if(perm) permute_lane = !permute_lane;                                                                                                                                                                                                
+
+     if ( permute_lane ) {
+       stmp(0) = vals[s](1);
+       stmp(1) = vals[s](0);
+       vals[s] = stmp;
+     }
+   }
+   merge(into,vals);
+ }
+
+
+ template<typename SiteSpinorType>
+ const SiteSpinorType & GparityGetChi(int &g, SiteSpinorType const* in, const int dir, const int f, StencilEntry *SE, StencilImpl &st){
+   const int mmu = dir % 4;
+   const int direction = st._directions[dir];
+   const int sl = st._grid->_simd_layout[direction];
+   const int perm = SE->_permute;
+   g = f;
+   
+   if(SE->_around_the_world && Params.twists[mmu]){
+     if(sl == 1){ //not SIMD vectorized in G-parity direction so just change the flavor index accessed to implement the twist
+       g = (f+1) % 2;
+       return in[SE->_offset];
+     }else{ //SIMD vectorized in Gparity direction
+       const int me = omp_get_thread_num();
+       const int distance = st._distances[dir];
+       assert(distance == -1 || distance == 1);
+       SiteSpinorType &tmp = GparityWilsonImpl_helper::getAB<SiteSpinorType, SiteSpinor, SiteHalfSpinor>::ref(tmp_full[me], tmp_half[me]);
+       GparityTwistPermute<SiteSpinorType>(tmp, in[SE->_offset], direction, distance, perm, st._grid);
+       return tmp;
+     }
+   }else return in[SE->_offset];
+ }
+
+
+ 
  inline void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
  {
    conformable(Uds._grid,GaugeGrid);
diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index 96b8ab0a..866e30d2 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -30,60 +30,77 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 #define REGISTER
 
-#define LOAD_CHIMU \
-  {const SiteSpinor & ref (in._odata[offset]);	\
-    Chimu_00=ref()(0)(0);\
-    Chimu_01=ref()(0)(1);\
-    Chimu_02=ref()(0)(2);\
-    Chimu_10=ref()(1)(0);\
-    Chimu_11=ref()(1)(1);\
-    Chimu_12=ref()(1)(2);\
-    Chimu_20=ref()(2)(0);\
-    Chimu_21=ref()(2)(1);\
-    Chimu_22=ref()(2)(2);\
-    Chimu_30=ref()(3)(0);\
-    Chimu_31=ref()(3)(1);\
-    Chimu_32=ref()(3)(2);}
+#define LOAD_CHIMU_BODY(F)			\
+  Chimu_00=ref(F)(0)(0);			\
+  Chimu_01=ref(F)(0)(1);			\
+  Chimu_02=ref(F)(0)(2);			\
+  Chimu_10=ref(F)(1)(0);			\
+  Chimu_11=ref(F)(1)(1);			\
+  Chimu_12=ref(F)(1)(2);			\
+  Chimu_20=ref(F)(2)(0);			\
+  Chimu_21=ref(F)(2)(1);			\
+  Chimu_22=ref(F)(2)(2);			\
+  Chimu_30=ref(F)(3)(0);			\
+  Chimu_31=ref(F)(3)(1);			\
+  Chimu_32=ref(F)(3)(2)
+
+#define LOAD_CHIMU(DIR,F)						\
+  { const SiteSpinor & ref (in._odata[offset]); LOAD_CHIMU_BODY(F); }
+
+#define LOAD_CHIMU_GPARITY(DIR,F)					\
+  { int g; const SiteSpinor & ref = GparityGetChi<SiteSpinor>(g,in._odata.data(),DIR,F,SE,st); LOAD_CHIMU_BODY(g); }
+
+#define LOAD_CHI_BODY(F)				\
+    Chi_00 = ref(F)(0)(0);\
+    Chi_01 = ref(F)(0)(1);\
+    Chi_02 = ref(F)(0)(2);\
+    Chi_10 = ref(F)(1)(0);\
+    Chi_11 = ref(F)(1)(1);\
+    Chi_12 = ref(F)(1)(2)
+
+#define LOAD_CHI(DIR,F)						\
+  {const SiteHalfSpinor &ref(buf[offset]); LOAD_CHI_BODY(F); }
+
+#define LOAD_CHI_GPARITY(DIR,F)						\
+  { int g; const SiteHalfSpinor &ref = GparityGetChi<SiteHalfSpinor>(g,buf,DIR,F,SE,st); LOAD_CHI_BODY(g); }
 
-#define LOAD_CHI\
-  {const SiteHalfSpinor &ref(buf[offset]);	\
-    Chi_00 = ref()(0)(0);\
-    Chi_01 = ref()(0)(1);\
-    Chi_02 = ref()(0)(2);\
-    Chi_10 = ref()(1)(0);\
-    Chi_11 = ref()(1)(1);\
-    Chi_12 = ref()(1)(2);}
 
 // To splat or not to splat depends on the implementation
-#define MULT_2SPIN(A)\
-  {auto & ref(U._odata[sU](A));			\
-   Impl::loadLinkElement(U_00,ref()(0,0));	\
-   Impl::loadLinkElement(U_10,ref()(1,0));	\
-   Impl::loadLinkElement(U_20,ref()(2,0));	\
-   Impl::loadLinkElement(U_01,ref()(0,1));	\
-   Impl::loadLinkElement(U_11,ref()(1,1));	\
-   Impl::loadLinkElement(U_21,ref()(2,1));	\
-    UChi_00 = U_00*Chi_00;\
-    UChi_10 = U_00*Chi_10;\
-    UChi_01 = U_10*Chi_00;\
-    UChi_11 = U_10*Chi_10;\
-    UChi_02 = U_20*Chi_00;\
-    UChi_12 = U_20*Chi_10;\
-    UChi_00+= U_01*Chi_01;\
-    UChi_10+= U_01*Chi_11;\
-    UChi_01+= U_11*Chi_01;\
-    UChi_11+= U_11*Chi_11;\
-    UChi_02+= U_21*Chi_01;\
-    UChi_12+= U_21*Chi_11;\
-    Impl::loadLinkElement(U_00,ref()(0,2));	\
-    Impl::loadLinkElement(U_10,ref()(1,2));	\
-    Impl::loadLinkElement(U_20,ref()(2,2));	\
-    UChi_00+= U_00*Chi_02;\
-    UChi_10+= U_00*Chi_12;\
-    UChi_01+= U_10*Chi_02;\
-    UChi_11+= U_10*Chi_12;\
-    UChi_02+= U_20*Chi_02;\
-    UChi_12+= U_20*Chi_12;}
+#define MULT_2SPIN_BODY \
+  Impl::loadLinkElement(U_00,ref()(0,0));	\
+  Impl::loadLinkElement(U_10,ref()(1,0));	\
+  Impl::loadLinkElement(U_20,ref()(2,0));	\
+  Impl::loadLinkElement(U_01,ref()(0,1));	\
+  Impl::loadLinkElement(U_11,ref()(1,1));	\
+  Impl::loadLinkElement(U_21,ref()(2,1));	\
+  UChi_00 = U_00*Chi_00;			\
+  UChi_10 = U_00*Chi_10;			\
+  UChi_01 = U_10*Chi_00;			\
+  UChi_11 = U_10*Chi_10;			\
+  UChi_02 = U_20*Chi_00;			\
+  UChi_12 = U_20*Chi_10;			\
+  UChi_00+= U_01*Chi_01;			\
+  UChi_10+= U_01*Chi_11;			\
+  UChi_01+= U_11*Chi_01;			\
+  UChi_11+= U_11*Chi_11;			\
+  UChi_02+= U_21*Chi_01;			\
+  UChi_12+= U_21*Chi_11;			\
+  Impl::loadLinkElement(U_00,ref()(0,2));	\
+  Impl::loadLinkElement(U_10,ref()(1,2));	\
+  Impl::loadLinkElement(U_20,ref()(2,2));	\
+  UChi_00+= U_00*Chi_02;			\
+  UChi_10+= U_00*Chi_12;			\
+  UChi_01+= U_10*Chi_02;			\
+  UChi_11+= U_10*Chi_12;			\
+  UChi_02+= U_20*Chi_02;			\
+  UChi_12+= U_20*Chi_12
+
+
+#define MULT_2SPIN(A,F)					\
+  {auto & ref(U._odata[sU](A)); MULT_2SPIN_BODY; }
+
+#define MULT_2SPIN_GPARITY(A,F)				\
+  {auto & ref(U._odata[sU](F)(A)); MULT_2SPIN_BODY; }
 
 
 #define PERMUTE_DIR(dir)			\
@@ -307,84 +324,85 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   result_31-= UChi_11;	\
   result_32-= UChi_12;
 
-#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON)	\
+#define HAND_STENCIL_LEG(PROJ,PERM,DIR,RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \
   SE=st.GetEntry(ptype,DIR,ss);			\
   offset = SE->_offset;				\
   local  = SE->_is_local;			\
   perm   = SE->_permute;			\
   if ( local ) {				\
-    LOAD_CHIMU;					\
+    LOAD_CHIMU_IMPL(DIR,F);			\
     PROJ;					\
     if ( perm) {				\
       PERMUTE_DIR(PERM);			\
     }						\
   } else {					\
-    LOAD_CHI;					\
+    LOAD_CHI_IMPL(DIR,F);			\
   }						\
-  MULT_2SPIN(DIR);				\
+  MULT_2SPIN_IMPL(DIR,F);			\
   RECON;					
 
-#define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON)	\
+
+#define HAND_STENCIL_LEG_INT(PROJ,PERM,DIR,RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL)	\
   SE=st.GetEntry(ptype,DIR,ss);			\
   offset = SE->_offset;				\
   local  = SE->_is_local;			\
   perm   = SE->_permute;			\
   if ( local ) {				\
-    LOAD_CHIMU;					\
+    LOAD_CHIMU_IMPL(DIR,F);			\
     PROJ;					\
     if ( perm) {				\
       PERMUTE_DIR(PERM);			\
     }						\
   } else if ( st.same_node[DIR] ) {		\
-    LOAD_CHI;					\
+    LOAD_CHI_IMPL(DIR,F);			\
   }						\
   if (local || st.same_node[DIR] ) {		\
-    MULT_2SPIN(DIR);				\
+    MULT_2SPIN_IMPL(DIR,F);			\
     RECON;					\
   }
 
-#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON)	\
+#define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL)	\
   SE=st.GetEntry(ptype,DIR,ss);			\
   offset = SE->_offset;				\
   if((!SE->_is_local)&&(!st.same_node[DIR]) ) {	\
-    LOAD_CHI;					\
-    MULT_2SPIN(DIR);				\
+    LOAD_CHI_IMPL(DIR,F);			\
+    MULT_2SPIN_IMPL(DIR,F);			\
     RECON;					\
     nmu++;					\
   }
 
-#define HAND_RESULT(ss)				\
+#define HAND_RESULT(ss,F)			\
   {						\
     SiteSpinor & ref (out._odata[ss]);		\
-    vstream(ref()(0)(0),result_00);		\
-    vstream(ref()(0)(1),result_01);		\
-    vstream(ref()(0)(2),result_02);		\
-    vstream(ref()(1)(0),result_10);		\
-    vstream(ref()(1)(1),result_11);		\
-    vstream(ref()(1)(2),result_12);		\
-    vstream(ref()(2)(0),result_20);		\
-    vstream(ref()(2)(1),result_21);		\
-    vstream(ref()(2)(2),result_22);		\
-    vstream(ref()(3)(0),result_30);		\
-    vstream(ref()(3)(1),result_31);		\
-    vstream(ref()(3)(2),result_32);		\
+    vstream(ref(F)(0)(0),result_00);		\
+    vstream(ref(F)(0)(1),result_01);		\
+    vstream(ref(F)(0)(2),result_02);		\
+    vstream(ref(F)(1)(0),result_10);		\
+    vstream(ref(F)(1)(1),result_11);		\
+    vstream(ref(F)(1)(2),result_12);		\
+    vstream(ref(F)(2)(0),result_20);		\
+    vstream(ref(F)(2)(1),result_21);		\
+    vstream(ref(F)(2)(2),result_22);		\
+    vstream(ref(F)(3)(0),result_30);		\
+    vstream(ref(F)(3)(1),result_31);		\
+    vstream(ref(F)(3)(2),result_32);		\
   }
 
-#define HAND_RESULT_EXT(ss)			\
+#define HAND_RESULT_EXT(ss,F)			\
   if (nmu){					\
     SiteSpinor & ref (out._odata[ss]);		\
-    ref()(0)(0)+=result_00;		\
-    ref()(0)(1)+=result_01;		\
-    ref()(0)(2)+=result_02;		\
-    ref()(1)(0)+=result_10;		\
-    ref()(1)(1)+=result_11;		\
-    ref()(1)(2)+=result_12;		\
-    ref()(2)(0)+=result_20;		\
-    ref()(2)(1)+=result_21;		\
-    ref()(2)(2)+=result_22;		\
-    ref()(3)(0)+=result_30;		\
-    ref()(3)(1)+=result_31;		\
-    ref()(3)(2)+=result_32;		\
+    ref(F)(0)(0)+=result_00;		\
+    ref(F)(0)(1)+=result_01;		\
+    ref(F)(0)(2)+=result_02;		\
+    ref(F)(1)(0)+=result_10;		\
+    ref(F)(1)(1)+=result_11;		\
+    ref(F)(1)(2)+=result_12;		\
+    ref(F)(2)(0)+=result_20;		\
+    ref(F)(2)(1)+=result_21;		\
+    ref(F)(2)(2)+=result_22;		\
+    ref(F)(3)(0)+=result_30;		\
+    ref(F)(3)(1)+=result_31;		\
+    ref(F)(3)(2)+=result_32;		\
   }
 
 
@@ -463,15 +481,18 @@ WilsonKernels<Impl>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGauge
   int offset,local,perm, ptype;
   StencilEntry *SE;
 
-  HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON);
-  HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM);
-  HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM);
-  HAND_RESULT(ss);
+#define HAND_DOP_SITE(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \
+  HAND_STENCIL_LEG(XM_PROJ,3,Xp,XM_RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(YM_PROJ,2,Yp,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL);	\
+  HAND_STENCIL_LEG(ZM_PROJ,1,Zp,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(TM_PROJ,0,Tp,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(XP_PROJ,3,Xm,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(YP_PROJ,2,Ym,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(ZP_PROJ,1,Zm,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(TP_PROJ,0,Tm,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_RESULT(ss,F)
+
+  HAND_DOP_SITE(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN);
 }
 
 template<class Impl>
@@ -485,16 +506,19 @@ void WilsonKernels<Impl>::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,Doub
 
   StencilEntry *SE;
   int offset,local,perm, ptype;
-  
-  HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON);
-  HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG(TP_PROJ,0,Tp,TP_RECON_ACCUM);
-  HAND_STENCIL_LEG(XM_PROJ,3,Xm,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG(YM_PROJ,2,Ym,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG(TM_PROJ,0,Tm,TM_RECON_ACCUM);
-  HAND_RESULT(ss);
+
+#define HAND_DOP_SITE_DAG(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \
+  HAND_STENCIL_LEG(XP_PROJ,3,Xp,XP_RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(YP_PROJ,2,Yp,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(ZP_PROJ,1,Zp,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(TP_PROJ,0,Tp,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(XM_PROJ,3,Xm,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(YM_PROJ,2,Ym,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(ZM_PROJ,1,Zm,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG(TM_PROJ,0,Tm,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_RESULT(ss,F)
+
+  HAND_DOP_SITE_DAG(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN);
 }
 
 template<class Impl> void 
@@ -509,16 +533,20 @@ WilsonKernels<Impl>::HandDhopSiteInt(StencilImpl &st,LebesgueOrder &lo,DoubledGa
 
   int offset,local,perm, ptype;
   StencilEntry *SE;
-  ZERO_RESULT;
-  HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(YM_PROJ,2,Yp,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(TM_PROJ,0,Tp,TM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(XP_PROJ,3,Xm,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(YP_PROJ,2,Ym,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(TP_PROJ,0,Tm,TP_RECON_ACCUM);
-  HAND_RESULT(ss);
+
+#define HAND_DOP_SITE_INT(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \
+  ZERO_RESULT; \
+  HAND_STENCIL_LEG_INT(XM_PROJ,3,Xp,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_INT(YM_PROJ,2,Yp,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_INT(TM_PROJ,0,Tp,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_INT(XP_PROJ,3,Xm,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_INT(YP_PROJ,2,Ym,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_INT(TP_PROJ,0,Tm,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_RESULT(ss,F)
+
+  HAND_DOP_SITE_INT(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN);
 }
 
 template<class Impl>
@@ -532,16 +560,20 @@ void WilsonKernels<Impl>::HandDhopSiteDagInt(StencilImpl &st,LebesgueOrder &lo,D
 
   StencilEntry *SE;
   int offset,local,perm, ptype;
-  ZERO_RESULT;
-  HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(TP_PROJ,0,Tp,TP_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(XM_PROJ,3,Xm,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(YM_PROJ,2,Ym,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG_INT(TM_PROJ,0,Tm,TM_RECON_ACCUM);
-  HAND_RESULT(ss);
+
+#define HAND_DOP_SITE_DAG_INT(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL)				\
+  ZERO_RESULT;							\
+  HAND_STENCIL_LEG_INT(XP_PROJ,3,Xp,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL);		\
+  HAND_STENCIL_LEG_INT(YP_PROJ,2,Yp,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL);		\
+  HAND_STENCIL_LEG_INT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL);		\
+  HAND_STENCIL_LEG_INT(TP_PROJ,0,Tp,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL);		\
+  HAND_STENCIL_LEG_INT(XM_PROJ,3,Xm,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL);		\
+  HAND_STENCIL_LEG_INT(YM_PROJ,2,Ym,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL);		\
+  HAND_STENCIL_LEG_INT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL);		\
+  HAND_STENCIL_LEG_INT(TM_PROJ,0,Tm,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL);		\
+  HAND_RESULT(ss,F)
+  
+  HAND_DOP_SITE_DAG_INT(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN);
 }
 
 template<class Impl> void 
@@ -557,16 +589,20 @@ WilsonKernels<Impl>::HandDhopSiteExt(StencilImpl &st,LebesgueOrder &lo,DoubledGa
   int offset,local,perm, ptype;
   StencilEntry *SE;
   int nmu=0;
-  ZERO_RESULT;
-  HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xp,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(YM_PROJ,2,Yp,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tp,TM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xm,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(YP_PROJ,2,Ym,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tm,TP_RECON_ACCUM);
-  HAND_RESULT_EXT(ss);
+
+#define HAND_DOP_SITE_EXT(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \
+  ZERO_RESULT; \
+  HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xp,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(YM_PROJ,2,Yp,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zp,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tp,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xm,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(YP_PROJ,2,Ym,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zm,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tm,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_RESULT_EXT(ss,F)
+
+  HAND_DOP_SITE_EXT(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN);
 }
 
 template<class Impl>
@@ -581,16 +617,20 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D
   StencilEntry *SE;
   int offset,local,perm, ptype;
   int nmu=0;
-  ZERO_RESULT;
-  HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(YP_PROJ,2,Yp,YP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tp,TP_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xm,XM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(YM_PROJ,2,Ym,YM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM);
-  HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tm,TM_RECON_ACCUM);
-  HAND_RESULT_EXT(ss);
+
+#define HAND_DOP_SITE_DAG_EXT(F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL) \
+  ZERO_RESULT; \
+  HAND_STENCIL_LEG_EXT(XP_PROJ,3,Xp,XP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(YP_PROJ,2,Yp,YP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(ZP_PROJ,1,Zp,ZP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(TP_PROJ,0,Tp,TP_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(XM_PROJ,3,Xm,XM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(YM_PROJ,2,Ym,YM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(ZM_PROJ,1,Zm,ZM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_STENCIL_LEG_EXT(TM_PROJ,0,Tm,TM_RECON_ACCUM,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL); \
+  HAND_RESULT_EXT(ss,F)
+
+  HAND_DOP_SITE_DAG_EXT(, LOAD_CHI,LOAD_CHIMU,MULT_2SPIN);
 }
 
   ////////////////////////////////////////////////
@@ -647,10 +687,130 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D
 				    FermionField &out){ assert(0); }	\
 
   HAND_SPECIALISE_EMPTY(GparityWilsonImplF);
-  HAND_SPECIALISE_EMPTY(GparityWilsonImplD);
+  //HAND_SPECIALISE_EMPTY(GparityWilsonImplD);
   HAND_SPECIALISE_EMPTY(GparityWilsonImplFH);
   HAND_SPECIALISE_EMPTY(GparityWilsonImplDF);
 
+
+
+
+
+template<> void 
+WilsonKernels<GparityWilsonImplD>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
+					  int ss,int sU,const FermionField &in, FermionField &out)
+{
+// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
+  typedef GparityWilsonImplD Impl;
+  typedef typename Simd::scalar_type S;
+  typedef typename Simd::vector_type V;
+
+  HAND_DECLARATIONS(ignore);
+
+  int offset,local,perm, ptype;
+  StencilEntry *SE;
+  HAND_DOP_SITE(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+  HAND_DOP_SITE(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+}
+
+template<>
+void WilsonKernels<GparityWilsonImplD>::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+						  int ss,int sU,const FermionField &in, FermionField &out)
+{
+  typedef GparityWilsonImplD Impl;
+  typedef typename Simd::scalar_type S;
+  typedef typename Simd::vector_type V;
+
+  HAND_DECLARATIONS(ignore);
+
+  StencilEntry *SE;
+  int offset,local,perm, ptype;
+  HAND_DOP_SITE_DAG(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+  HAND_DOP_SITE_DAG(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+}
+
+template<> void 
+WilsonKernels<GparityWilsonImplD>::HandDhopSiteInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
+					  int ss,int sU,const FermionField &in, FermionField &out)
+{
+// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
+  typedef GparityWilsonImplD Impl;
+  typedef typename Simd::scalar_type S;
+  typedef typename Simd::vector_type V;
+
+  HAND_DECLARATIONS(ignore);
+
+  int offset,local,perm, ptype;
+  StencilEntry *SE;
+  HAND_DOP_SITE_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+  HAND_DOP_SITE_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+}
+
+template<>
+void WilsonKernels<GparityWilsonImplD>::HandDhopSiteDagInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+						  int ss,int sU,const FermionField &in, FermionField &out)
+{
+  typedef GparityWilsonImplD Impl;
+  typedef typename Simd::scalar_type S;
+  typedef typename Simd::vector_type V;
+
+  HAND_DECLARATIONS(ignore);
+
+  StencilEntry *SE;
+  int offset,local,perm, ptype;  
+  HAND_DOP_SITE_DAG_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+  HAND_DOP_SITE_DAG_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+}
+
+template<> void 
+WilsonKernels<GparityWilsonImplD>::HandDhopSiteExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
+					  int ss,int sU,const FermionField &in, FermionField &out)
+{
+// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
+  typedef GparityWilsonImplD Impl;
+  typedef typename Simd::scalar_type S;
+  typedef typename Simd::vector_type V;
+
+  HAND_DECLARATIONS(ignore);
+
+  int offset,local,perm, ptype;
+  StencilEntry *SE;
+  int nmu=0;
+  HAND_DOP_SITE_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+  nmu = 0;
+  HAND_DOP_SITE_EXT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);  
+}
+
+template<>
+void WilsonKernels<GparityWilsonImplD>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
+						  int ss,int sU,const FermionField &in, FermionField &out)
+{
+  typedef GparityWilsonImplD Impl;
+  typedef typename Simd::scalar_type S;
+  typedef typename Simd::vector_type V;
+
+  HAND_DECLARATIONS(ignore);
+
+  StencilEntry *SE;
+  int offset,local,perm, ptype;
+  int nmu=0;
+  HAND_DOP_SITE_DAG_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+  nmu = 0;
+  HAND_DOP_SITE_DAG_EXT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
 ////////////// Wilson ; uses this implementation /////////////////////
 
 #define INSTANTIATE_THEM(A) \
diff --git a/tests/core/Test_gparity.cc b/tests/core/Test_gparity.cc
index cfb5d2c3..81091e9e 100644
--- a/tests/core/Test_gparity.cc
+++ b/tests/core/Test_gparity.cc
@@ -33,22 +33,68 @@ using namespace std;
 using namespace Grid;
 using namespace Grid::QCD;
 
-typedef typename GparityDomainWallFermionR::FermionField FermionField;
+//typedef GparityDomainWallFermionD GparityDiracOp;
+//typedef DomainWallFermionD StandardDiracOp;
+//#define DOP_PARAMS
 
+typedef GparityMobiusFermionD GparityDiracOp;
+typedef MobiusFermionD StandardDiracOp;
+#define DOP_PARAMS ,1.5, 0.5
+
+
+typedef typename GparityDiracOp::FermionField GparityFermionField;
+typedef typename GparityDiracOp::GaugeField GparityGaugeField;
+typedef typename GparityFermionField::vector_type vComplexType;
+
+typedef typename StandardDiracOp::FermionField StandardFermionField;
+typedef typename StandardDiracOp::GaugeField StandardGaugeField;
+
+enum{ same_vComplex = std::is_same<vComplexType, typename StandardFermionField::vector_type>::value };
+static_assert(same_vComplex == 1, "Dirac Operators must have same underlying SIMD complex type");
 
 int main (int argc, char ** argv)
 {
-  const int nu = 3;
+  int nu = 0;
 
   Grid_init(&argc,&argv);
 
+  for(int i=1;i<argc;i++){
+    if(std::string(argv[i]) == "--Gparity-dir"){
+      std::stringstream ss; ss << argv[i+1]; ss >> nu;
+      std::cout << GridLogMessage << "Set Gparity direction to " << nu << std::endl;
+    }
+  }
+
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Testing Gparity Dirac operator                  "<<std::endl;
+  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplexType::Nsimd()<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using UNROLLED Nc=3       WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
 
   const int Ls=4;
-  const int L =4;
-  std::vector<int> latt_2f(Nd,L);
-  std::vector<int> latt_1f(Nd,L); latt_1f[nu] = 2*L;
+  //const int L =4;
+  //std::vector<int> latt_2f(Nd,L);
 
-  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> latt_2f = GridDefaultLatt();
+  std::vector<int> latt_1f(latt_2f); latt_1f[nu] = 2*latt_2f[nu];
+  int L = latt_2f[nu];
+
+
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplexType::Nsimd());
+
+  std::cout << GridLogMessage << "SIMD layout: ";
+  for(int i=0;i<simd_layout.size();i++) std::cout << simd_layout[i] << " ";
+  std::cout << std::endl;
+  
   std::vector<int> mpi_layout  = GridDefaultMpi(); //node layout
 
   GridCartesian         * UGrid_1f   = SpaceTimeGrid::makeFourDimGrid(latt_1f, simd_layout, mpi_layout);
@@ -67,13 +113,13 @@ int main (int argc, char ** argv)
   GridParallelRNG          RNG5_2f(FGrid_2f);  RNG5_2f.SeedFixedIntegers(seeds5);
   GridParallelRNG          RNG4_2f(UGrid_2f);  RNG4_2f.SeedFixedIntegers(seeds4);
 
-  LatticeGaugeField Umu_2f(UGrid_2f);
+  GparityGaugeField Umu_2f(UGrid_2f);
   SU3::HotConfiguration(RNG4_2f,Umu_2f);
 
-  LatticeFermion    src   (FGrid_2f); 
-  LatticeFermion    tmpsrc(FGrid_2f); 
-  FermionField      src_2f(FGrid_2f); 
-  LatticeFermion    src_1f(FGrid_1f); 
+  StandardFermionField    src   (FGrid_2f); 
+  StandardFermionField    tmpsrc(FGrid_2f); 
+  GparityFermionField      src_2f(FGrid_2f); 
+  StandardFermionField    src_1f(FGrid_1f); 
 
   // Replicate fermion source
   random(RNG5_2f,src);
@@ -81,8 +127,8 @@ int main (int argc, char ** argv)
   tmpsrc=src*2.0;
   PokeIndex<0>(src_2f,tmpsrc,1);
 
-  LatticeFermion result_1f(FGrid_1f); result_1f=zero;
-  LatticeGaugeField Umu_1f(UGrid_1f); 
+  StandardFermionField result_1f(FGrid_1f); result_1f=zero;
+  StandardGaugeField Umu_1f(UGrid_1f); 
   Replicate(Umu_2f,Umu_1f);
 
   //Coordinate grid for reference
@@ -92,7 +138,7 @@ int main (int argc, char ** argv)
   //Copy-conjugate the gauge field
   //First C-shift the lattice by Lx/2
   {
-    LatticeGaugeField Umu_shift = conjugate( Cshift(Umu_1f,nu,L) );
+    StandardGaugeField Umu_shift = conjugate( Cshift(Umu_1f,nu,L) );
     Umu_1f = where( xcoor_1f >= Integer(L), Umu_shift, Umu_1f );
 
     // hack test to check the same
@@ -101,7 +147,7 @@ int main (int argc, char ** argv)
     cout << GridLogMessage << "Umu diff " << norm2(Umu_shift)<<std::endl;
 
     //Make the gauge field antiperiodic in nu-direction
-    LatticeColourMatrix Unu(UGrid_1f);
+    decltype(PeekIndex<LorentzIndex>(Umu_1f,nu)) Unu(UGrid_1f);
     Unu = PeekIndex<LorentzIndex>(Umu_1f,nu);
     Unu = where(xcoor_1f == Integer(2*L-1), -Unu, Unu);
     PokeIndex<LorentzIndex>(Umu_1f,Unu,nu);
@@ -115,33 +161,33 @@ int main (int argc, char ** argv)
 
   RealD mass=0.0;
   RealD M5=1.8;
-  DomainWallFermionR Ddwf(Umu_1f,*FGrid_1f,*FrbGrid_1f,*UGrid_1f,*UrbGrid_1f,mass,M5);
+  StandardDiracOp Ddwf(Umu_1f,*FGrid_1f,*FrbGrid_1f,*UGrid_1f,*UrbGrid_1f,mass,M5 DOP_PARAMS);
 
-  LatticeFermion    src_o_1f(FrbGrid_1f);
-  LatticeFermion result_o_1f(FrbGrid_1f);
+  StandardFermionField    src_o_1f(FrbGrid_1f);
+  StandardFermionField result_o_1f(FrbGrid_1f);
   pickCheckerboard(Odd,src_o_1f,src_1f);
   result_o_1f=zero;
 
-  SchurDiagMooeeOperator<DomainWallFermionR,LatticeFermion> HermOpEO(Ddwf);
-  ConjugateGradient<LatticeFermion> CG(1.0e-8,10000);
+  SchurDiagMooeeOperator<StandardDiracOp,StandardFermionField> HermOpEO(Ddwf);
+  ConjugateGradient<StandardFermionField> CG(1.0e-8,10000);
   CG(HermOpEO,src_o_1f,result_o_1f);
   
   //  const int nu = 3;
   std::vector<int> twists(Nd,0);
   twists[nu] = 1;
-  GparityDomainWallFermionR::ImplParams params;
+  GparityDiracOp::ImplParams params;
   params.twists = twists;
-  GparityDomainWallFermionR GPDdwf(Umu_2f,*FGrid_2f,*FrbGrid_2f,*UGrid_2f,*UrbGrid_2f,mass,M5,params);
+  GparityDiracOp GPDdwf(Umu_2f,*FGrid_2f,*FrbGrid_2f,*UGrid_2f,*UrbGrid_2f,mass,M5 DOP_PARAMS,params);
 
   for(int disp=-1;disp<=1;disp+=2)
   for(int mu=0;mu<5;mu++)
   { 
-    FermionField Dsrc_2f(FGrid_2f);
+    GparityFermionField Dsrc_2f(FGrid_2f);
 
-    LatticeFermion Dsrc_1f(FGrid_1f);
-    LatticeFermion Dsrc_2freplica(FGrid_1f);
-    LatticeFermion Dsrc_2freplica0(FGrid_1f);
-    LatticeFermion Dsrc_2freplica1(FGrid_1f);
+    StandardFermionField Dsrc_1f(FGrid_1f);
+    StandardFermionField Dsrc_2freplica(FGrid_1f);
+    StandardFermionField Dsrc_2freplica0(FGrid_1f);
+    StandardFermionField Dsrc_2freplica1(FGrid_1f);
 
     if ( mu ==0 ) {
       std::cout << GridLogMessage<< " Cross checking entire hopping term"<<std::endl;
@@ -156,8 +202,8 @@ int main (int argc, char ** argv)
     std::cout << GridLogMessage << "S norms "<< norm2(src_2f) << " " << norm2(src_1f)  <<std::endl;
     std::cout << GridLogMessage << "D norms "<< norm2(Dsrc_2f)<< " " << norm2(Dsrc_1f) <<std::endl;
 
-    LatticeFermion Dsrc_2f0(FGrid_2f); Dsrc_2f0 = PeekIndex<0>(Dsrc_2f,0);
-    LatticeFermion Dsrc_2f1(FGrid_2f); Dsrc_2f1 = PeekIndex<0>(Dsrc_2f,1);
+    StandardFermionField Dsrc_2f0(FGrid_2f); Dsrc_2f0 = PeekIndex<0>(Dsrc_2f,0);
+    StandardFermionField Dsrc_2f1(FGrid_2f); Dsrc_2f1 = PeekIndex<0>(Dsrc_2f,1);
 
     //    Dsrc_2f1 = Dsrc_2f1 - Dsrc_2f0;
     //    std::cout << GridLogMessage << " Cross check two halves " <<norm2(Dsrc_2f1)<<std::endl;
@@ -174,20 +220,20 @@ int main (int argc, char ** argv)
   }
 
   {
-    FermionField chi   (FGrid_2f); gaussian(RNG5_2f,chi);
-    FermionField phi   (FGrid_2f); gaussian(RNG5_2f,phi);
+    GparityFermionField chi   (FGrid_2f); gaussian(RNG5_2f,chi);
+    GparityFermionField phi   (FGrid_2f); gaussian(RNG5_2f,phi);
   
-    FermionField chi_e   (FrbGrid_2f);
-    FermionField chi_o   (FrbGrid_2f);
+    GparityFermionField chi_e   (FrbGrid_2f);
+    GparityFermionField chi_o   (FrbGrid_2f);
     
-    FermionField dchi_e  (FrbGrid_2f);
-    FermionField dchi_o  (FrbGrid_2f);
+    GparityFermionField dchi_e  (FrbGrid_2f);
+    GparityFermionField dchi_o  (FrbGrid_2f);
     
-    FermionField phi_e   (FrbGrid_2f);
-    FermionField phi_o   (FrbGrid_2f);
+    GparityFermionField phi_e   (FrbGrid_2f);
+    GparityFermionField phi_o   (FrbGrid_2f);
     
-    FermionField dphi_e  (FrbGrid_2f);
-    FermionField dphi_o  (FrbGrid_2f);
+    GparityFermionField dphi_e  (FrbGrid_2f);
+    GparityFermionField dphi_o  (FrbGrid_2f);
 
     pickCheckerboard(Even,chi_e,chi);
     pickCheckerboard(Odd ,chi_o,chi);
@@ -212,14 +258,14 @@ int main (int argc, char ** argv)
 
   }
 
-  FermionField result_2f(FGrid_2f); result_2f=zero;
-  FermionField    src_o_2f(FrbGrid_2f);
-  FermionField result_o_2f(FrbGrid_2f);
+  GparityFermionField result_2f(FGrid_2f); result_2f=zero;
+  GparityFermionField    src_o_2f(FrbGrid_2f);
+  GparityFermionField result_o_2f(FrbGrid_2f);
   pickCheckerboard(Odd,src_o_2f,src_2f);
   result_o_2f=zero;
 
-  ConjugateGradient<FermionField> CG2f(1.0e-8,10000);
-  SchurDiagMooeeOperator<GparityDomainWallFermionR,FermionField> HermOpEO2f(GPDdwf);
+  ConjugateGradient<GparityFermionField> CG2f(1.0e-8,10000);
+  SchurDiagMooeeOperator<GparityDiracOp,GparityFermionField> HermOpEO2f(GPDdwf);
   CG2f(HermOpEO2f,src_o_2f,result_o_2f);
 
   std::cout << "2f cb "<<result_o_2f.checkerboard<<std::endl;
@@ -227,10 +273,10 @@ int main (int argc, char ** argv)
 
   std::cout << " result norms " <<norm2(result_o_2f)<<" " <<norm2(result_o_1f)<<std::endl;
 
-  LatticeFermion    res0o  (FrbGrid_2f); 
-  LatticeFermion    res1o  (FrbGrid_2f); 
-  LatticeFermion    res0  (FGrid_2f); 
-  LatticeFermion    res1  (FGrid_2f); 
+  StandardFermionField    res0o  (FrbGrid_2f); 
+  StandardFermionField    res1o  (FrbGrid_2f); 
+  StandardFermionField    res0  (FGrid_2f); 
+  StandardFermionField    res1  (FGrid_2f); 
 
   res0=zero;
   res1=zero;
@@ -244,9 +290,9 @@ int main (int argc, char ** argv)
   setCheckerboard(res0,res0o);
   setCheckerboard(res1,res1o);
 
-  LatticeFermion replica (FGrid_1f);
-  LatticeFermion replica0(FGrid_1f);
-  LatticeFermion replica1(FGrid_1f);
+  StandardFermionField replica (FGrid_1f);
+  StandardFermionField replica0(FGrid_1f);
+  StandardFermionField replica1(FGrid_1f);
   Replicate(res0,replica0);
   Replicate(res1,replica1);
 

From 061e48fd73c63d924e8d777ab74287f7d69d7c2a Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Tue, 22 Aug 2017 18:12:12 -0400
Subject: [PATCH 057/115] Replaced slow unpack-repack in G-parity BC twist with
 intrinsics version

---
 lib/qcd/action/fermion/FermionOperatorImpl.h | 26 ++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/lib/qcd/action/fermion/FermionOperatorImpl.h b/lib/qcd/action/fermion/FermionOperatorImpl.h
index 1e344521..5300063b 100644
--- a/lib/qcd/action/fermion/FermionOperatorImpl.h
+++ b/lib/qcd/action/fermion/FermionOperatorImpl.h
@@ -565,6 +565,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
 
  template<typename SiteSpinorType>
  void GparityTwistPermute(SiteSpinorType &into,  const SiteSpinorType &from, const int direction, const int distance, const int perm, GridBase* grid){
+#if 0
    typedef typename SiteSpinorType::scalar_object sobj;
    sobj stmp;
    std::vector<sobj> vals(grid->Nsimd());
@@ -589,6 +590,31 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
      }
    }
    merge(into,vals);
+#else
+   int permute_type = grid->PermuteType(direction);
+   typedef typename SiteSpinorType::vector_type vtype;
+   vtype tmp1, tmp2, tmp3, tmp4;
+
+   for(int s=0;s<Ns;s++){
+     for(int c=0;c<Nrepresentation;c++){
+       permute(tmp1, from(1)(s)(c), permute_type); //1l 1h -> 1h 1l
+       exchange(tmp2,tmp3, from(0)(s)(c), tmp1, permute_type); // 0l 0h , 1h 1l -> 0l 1h 0h,1l
+       permute(tmp4, tmp3, permute_type); //0h,1l -> 1l,0h
+              
+       if( (distance == 1 && !perm) || (distance == -1 && perm) ){
+	 //Pulled fermion through forwards face, GPBC on upper component
+	 //Need 0= 0l 1h   1= 1l 0h
+	 into(0)(s)(c) = tmp2;
+	 into(1)(s)(c) = tmp4;
+       }else if( (distance == -1 && !perm) || (distance == 1 && perm) ){
+	 //Pulled fermion through backwards face, GPBC on lower component
+	 //Need 0= 1l 0h   1= 0l 1h	 
+	 into(0)(s)(c) = tmp4;
+	 into(1)(s)(c) = tmp2;
+       }else assert(0);
+     }
+   }
+#endif
  }
 
 

From 459f70e8d4a578ee9d176c5c068dfcb924b34866 Mon Sep 17 00:00:00 2001
From: David Murphy <djm2131@columbia.edu>
Date: Tue, 22 Aug 2017 22:38:30 -0400
Subject: [PATCH 058/115] Check-in of working Mobius EOFA class and tests

---
 lib/qcd/action/fermion/Fermion.h              |   6 +-
 lib/qcd/action/fermion/MobiusEOFAFermion.cc   |  21 +-
 lib/qcd/action/fermion/MobiusEOFAFermion.h    |   4 +-
 .../action/fermion/MobiusEOFAFermioncache.cc  |  24 +-
 .../action/fermion/MobiusEOFAFermiondense.cc  | 184 +++++
 .../action/fermion/MobiusEOFAFermionssp.cc    | 290 ++++++++
 .../action/fermion/MobiusEOFAFermionvec.cc    | 654 ++++++++++++++++++
 tests/core/Test_mobius_eofa_even_odd.cc       | 241 +++++++
 tests/debug/Test_heatbath_mobius_eofa.cc      | 104 +++
 .../Test_heatbath_mobius_eofa_gparity.cc      | 109 +++
 tests/debug/Test_reweight_dwf_eofa_gparity.cc |   2 +-
 tests/debug/Test_reweight_mobius_eofa.cc      | 215 ++++++
 .../Test_reweight_mobius_eofa_gparity.cc      | 218 ++++++
 tests/forces/Test_dwf_gpforce_eofa.cc         |   6 +-
 tests/forces/Test_mobius_force_eofa.cc        | 166 +++++
 tests/forces/Test_mobius_gpforce_eofa.cc      | 171 +++++
 16 files changed, 2383 insertions(+), 32 deletions(-)
 create mode 100644 lib/qcd/action/fermion/MobiusEOFAFermiondense.cc
 create mode 100644 lib/qcd/action/fermion/MobiusEOFAFermionssp.cc
 create mode 100644 lib/qcd/action/fermion/MobiusEOFAFermionvec.cc
 create mode 100644 tests/core/Test_mobius_eofa_even_odd.cc
 create mode 100644 tests/debug/Test_heatbath_mobius_eofa.cc
 create mode 100644 tests/debug/Test_heatbath_mobius_eofa_gparity.cc
 create mode 100644 tests/debug/Test_reweight_mobius_eofa.cc
 create mode 100644 tests/debug/Test_reweight_mobius_eofa_gparity.cc
 create mode 100644 tests/forces/Test_mobius_force_eofa.cc
 create mode 100644 tests/forces/Test_mobius_gpforce_eofa.cc

diff --git a/lib/qcd/action/fermion/Fermion.h b/lib/qcd/action/fermion/Fermion.h
index 99d97895..ad2f383d 100644
--- a/lib/qcd/action/fermion/Fermion.h
+++ b/lib/qcd/action/fermion/Fermion.h
@@ -265,9 +265,9 @@ typedef MobiusFermion<GparityWilsonImplRL> GparityMobiusFermionRL;
 typedef MobiusFermion<GparityWilsonImplFH> GparityMobiusFermionFH;
 typedef MobiusFermion<GparityWilsonImplDF> GparityMobiusFermionDF;
 
-typedef MobiusEOFAFermion<GparityWilsonImplR> GparityMobiusFermionR;
-typedef MobiusEOFAFermion<GparityWilsonImplF> GparityMobiusFermionF;
-typedef MobiusEOFAFermion<GparityWilsonImplD> GparityMobiusFermionD;
+typedef MobiusEOFAFermion<GparityWilsonImplR> GparityMobiusEOFAFermionR;
+typedef MobiusEOFAFermion<GparityWilsonImplF> GparityMobiusEOFAFermionF;
+typedef MobiusEOFAFermion<GparityWilsonImplD> GparityMobiusEOFAFermionD;
 
 typedef MobiusEOFAFermion<GparityWilsonImplRL> GparityMobiusEOFAFermionRL;
 typedef MobiusEOFAFermion<GparityWilsonImplFH> GparityMobiusEOFAFermionFH;
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermion.cc b/lib/qcd/action/fermion/MobiusEOFAFermion.cc
index c0837d78..085fa988 100644
--- a/lib/qcd/action/fermion/MobiusEOFAFermion.cc
+++ b/lib/qcd/action/fermion/MobiusEOFAFermion.cc
@@ -142,7 +142,7 @@ namespace QCD {
 
       RealD DtInv_p(0.0), DtInv_m(0.0);
       RealD N = std::pow(c+d,Ls) + m*std::pow(c-d,Ls);
-      FermionField tmp = zero;
+      FermionField tmp(this->FermionGrid());
 
       for(int s=0; s<Ls; ++s){
       for(int sp=0; sp<Ls; ++sp){
@@ -152,15 +152,14 @@ namespace QCD {
         DtInv_m = m * std::pow(-1.0,sp-s+1) * std::pow(c-d,Ls+sp-s) / std::pow(c+d,sp-s+1) / N;
         DtInv_m += (s > sp) ? 0.0 : std::pow(-1.0,sp-s) * std::pow(c-d,sp-s) / std::pow(c+d,sp-s+1);
 
-        if(dag){
-          RealD tmp(DtInv_p);
-          DtInv_p = DtInv_m;
-          DtInv_m = tmp;
+        if(sp == 0){
+          axpby_ssp_pplus (tmp, 0.0, tmp, DtInv_p, psi, s, sp);
+          axpby_ssp_pminus(tmp, 0.0, tmp, DtInv_m, psi, s, sp);
+        } else {
+          axpby_ssp_pplus (tmp, 1.0, tmp, DtInv_p, psi, s, sp);
+          axpby_ssp_pminus(tmp, 1.0, tmp, DtInv_m, psi, s, sp);
         }
 
-        axpby_ssp_pplus (tmp, 1.0, tmp, DtInv_p, psi, s, sp);
-        axpby_ssp_pminus(tmp, 1.0, tmp, DtInv_m, psi, s, sp);
-
       }}
     }
 
@@ -217,11 +216,11 @@ namespace QCD {
     template<class Impl>
     void MobiusEOFAFermion<Impl>::M5Ddag(const FermionField& psi, FermionField& chi)
     {
-      int   Ls    = this->Ls;
+      int Ls = this->Ls;
 
       std::vector<Coeff_t> diag(Ls,1.0);
-      std::vector<Coeff_t> upper(Ls,-1.0); upper[Ls-1] = this->mq1 + shiftp;
-      std::vector<Coeff_t> lower(Ls,-1.0); lower[0]    = this->mq1 + shiftm;
+      std::vector<Coeff_t> upper(Ls,-1.0);  upper[Ls-1] = this->mq1;
+      std::vector<Coeff_t> lower(Ls,-1.0);  lower[0]    = this->mq1;
 
       // no shift term
       if(this->shift == 0.0){ this->M5Ddag(psi, chi, chi, lower, diag, upper); }
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermion.h b/lib/qcd/action/fermion/MobiusEOFAFermion.h
index 0a8d1788..519b49e7 100644
--- a/lib/qcd/action/fermion/MobiusEOFAFermion.h
+++ b/lib/qcd/action/fermion/MobiusEOFAFermion.h
@@ -111,7 +111,7 @@ namespace QCD {
   };
 }}
 
-#define INSTANTIATE_DPERP_DWF_EOFA(A)\
+#define INSTANTIATE_DPERP_MOBIUS_EOFA(A)\
 template void MobiusEOFAFermion<A>::M5D(const FermionField& psi, const FermionField& phi, FermionField& chi, \
   std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper); \
 template void MobiusEOFAFermion<A>::M5D_shift(const FermionField& psi, const FermionField& phi, FermionField& chi, \
@@ -122,7 +122,7 @@ template void MobiusEOFAFermion<A>::M5Ddag_shift(const FermionField& psi, const
   std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper, std::vector<Coeff_t>& shift_coeffs); \
 template void MobiusEOFAFermion<A>::MooeeInv(const FermionField& psi, FermionField& chi); \
 template void MobiusEOFAFermion<A>::MooeeInv_shift(const FermionField& psi, FermionField& chi); \
-template void MobiusEOFAFermion<A>::MooeeInvDag(const FermionField& psi, FermionField& chi);
+template void MobiusEOFAFermion<A>::MooeeInvDag(const FermionField& psi, FermionField& chi); \
 template void MobiusEOFAFermion<A>::MooeeInvDag_shift(const FermionField& psi, FermionField& chi);
 
 #undef  MOBIUS_EOFA_DPERP_DENSE
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermioncache.cc b/lib/qcd/action/fermion/MobiusEOFAFermioncache.cc
index d184ebe2..420f6390 100644
--- a/lib/qcd/action/fermion/MobiusEOFAFermioncache.cc
+++ b/lib/qcd/action/fermion/MobiusEOFAFermioncache.cc
@@ -410,19 +410,19 @@ namespace QCD {
 
   #ifdef MOBIUS_EOFA_DPERP_CACHE
 
-  INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplF);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplD);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplF);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplD);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplF);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplD);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplD);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplD);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplD);
 
-  INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplFH);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplDF);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplFH);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplDF);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplFH);
-  INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplDF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplDF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplDF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplDF);
 
   #endif
 
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermiondense.cc b/lib/qcd/action/fermion/MobiusEOFAFermiondense.cc
new file mode 100644
index 00000000..d66b8cd9
--- /dev/null
+++ b/lib/qcd/action/fermion/MobiusEOFAFermiondense.cc
@@ -0,0 +1,184 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/MobiusEOFAFermiondense.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid_Eigen_Dense.h>
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/MobiusEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+  /*
+  * Dense matrix versions of routines
+  */
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInv(const FermionField& psi, FermionField& chi)
+  {
+    this->MooeeInternal(psi, chi, DaggerNo, InverseYes);
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInv_shift(const FermionField& psi, FermionField& chi)
+  {
+    this->MooeeInternal(psi, chi, DaggerNo, InverseYes);
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInvDag(const FermionField& psi, FermionField& chi)
+  {
+    this->MooeeInternal(psi, chi, DaggerYes, InverseYes);
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInvDag_shift(const FermionField& psi, FermionField& chi)
+  {
+    this->MooeeInternal(psi, chi, DaggerYes, InverseYes);
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv)
+  {
+    int Ls = this->Ls;
+    int LLs = psi._grid->_rdimensions[0];
+    int vol = psi._grid->oSites()/LLs;
+
+    int pm      = this->pm;
+    RealD shift = this->shift;
+    RealD alpha = this->alpha;
+    RealD k     = this->k;
+    RealD mq1   = this->mq1;
+
+    chi.checkerboard = psi.checkerboard;
+
+    assert(Ls==LLs);
+
+    Eigen::MatrixXd Pplus  = Eigen::MatrixXd::Zero(Ls,Ls);
+    Eigen::MatrixXd Pminus = Eigen::MatrixXd::Zero(Ls,Ls);
+
+    for(int s=0;s<Ls;s++){
+        Pplus(s,s)  = this->bee[s];
+        Pminus(s,s) = this->bee[s];
+    }
+
+    for(int s=0; s<Ls-1; s++){
+        Pminus(s,s+1) = -this->cee[s];
+    }
+
+    for(int s=0; s<Ls-1; s++){
+        Pplus(s+1,s) = -this->cee[s+1];
+    }
+    Pplus (0,Ls-1) = mq1*this->cee[0];
+    Pminus(Ls-1,0) = mq1*this->cee[Ls-1];
+
+    if(shift != 0.0){
+      Coeff_t N = 2.0 * ( std::pow(alpha+1.0,Ls) + mq1*std::pow(alpha-1.0,Ls) );
+      for(int s=0; s<Ls; ++s){
+        if(pm == 1){ Pplus(s,Ls-1) += shift * k * N * std::pow(-1.0,s) * std::pow(alpha-1.0,s) / std::pow(alpha+1.0,Ls+s+1); }
+        else{ Pminus(Ls-1-s,Ls-1) -= shift * k * N * std::pow(-1.0,s) * std::pow(alpha-1.0,s) / std::pow(alpha+1.0,Ls+s+1); }
+      }
+    }
+
+    Eigen::MatrixXd PplusMat ;
+    Eigen::MatrixXd PminusMat;
+
+    if(inv){
+      PplusMat  = Pplus.inverse();
+      PminusMat = Pminus.inverse();
+    } else {
+      PplusMat  = Pplus;
+      PminusMat = Pminus;
+    }
+
+    if(dag){
+      PplusMat.adjointInPlace();
+      PminusMat.adjointInPlace();
+    }
+
+    // For the non-vectorised s-direction this is simple
+
+    for(auto site=0; site<vol; site++){
+
+        SiteSpinor     SiteChi;
+        SiteHalfSpinor SitePplus;
+        SiteHalfSpinor SitePminus;
+
+        for(int s1=0; s1<Ls; s1++){
+            SiteChi = zero;
+            for(int s2=0; s2<Ls; s2++){
+                int lex2 = s2 + Ls*site;
+                if(PplusMat(s1,s2) != 0.0){
+                    spProj5p(SitePplus,psi[lex2]);
+                    accumRecon5p(SiteChi, PplusMat(s1,s2)*SitePplus);
+                }
+                if(PminusMat(s1,s2) != 0.0){
+                    spProj5m(SitePminus, psi[lex2]);
+                    accumRecon5m(SiteChi, PminusMat(s1,s2)*SitePminus);
+                }
+            }
+            chi[s1+Ls*site] = SiteChi*0.5;
+        }
+    }
+  }
+
+  #ifdef MOBIUS_EOFA_DPERP_DENSE
+
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplD);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplD);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplD);
+
+    template void MobiusEOFAFermion<GparityWilsonImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<GparityWilsonImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<WilsonImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<WilsonImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<ZWilsonImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<ZWilsonImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplDF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplDF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplDF);
+
+    template void MobiusEOFAFermion<GparityWilsonImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<GparityWilsonImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<WilsonImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<WilsonImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<ZWilsonImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<ZWilsonImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+
+  #endif
+
+}}
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermionssp.cc b/lib/qcd/action/fermion/MobiusEOFAFermionssp.cc
new file mode 100644
index 00000000..c86bb995
--- /dev/null
+++ b/lib/qcd/action/fermion/MobiusEOFAFermionssp.cc
@@ -0,0 +1,290 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/MobiusEOFAFermionssp.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/MobiusEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+  // FIXME -- make a version of these routines with site loop outermost for cache reuse.
+  // Pminus fowards
+  // Pplus  backwards
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5D(const FermionField& psi, const FermionField& phi,
+    FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+  {
+    Coeff_t one(1.0);
+    int Ls = this->Ls;
+    for(int s=0; s<Ls; s++){
+      if(s==0) {
+        axpby_ssp_pminus(chi, diag[s], phi, upper[s], psi, s, s+1);
+        axpby_ssp_pplus (chi, one, chi, lower[s], psi, s, Ls-1);
+      } else if (s==(Ls-1)) {
+        axpby_ssp_pminus(chi, diag[s], phi, upper[s], psi, s, 0);
+        axpby_ssp_pplus (chi, one, chi, lower[s], psi, s, s-1);
+      } else {
+        axpby_ssp_pminus(chi, diag[s], phi, upper[s], psi, s, s+1);
+        axpby_ssp_pplus(chi, one, chi, lower[s], psi, s, s-1);
+      }
+    }
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5D_shift(const FermionField& psi, const FermionField& phi,
+    FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper,
+    std::vector<Coeff_t>& shift_coeffs)
+  {
+    Coeff_t one(1.0);
+    int Ls = this->Ls;
+    for(int s=0; s<Ls; s++){
+      if(s==0) {
+        axpby_ssp_pminus(chi, diag[s], phi, upper[s], psi, s, s+1);
+        axpby_ssp_pplus (chi, one, chi, lower[s], psi, s, Ls-1);
+      } else if (s==(Ls-1)) {
+        axpby_ssp_pminus(chi, diag[s], phi, upper[s], psi, s, 0);
+        axpby_ssp_pplus (chi, one, chi, lower[s], psi, s, s-1);
+      } else {
+        axpby_ssp_pminus(chi, diag[s], phi, upper[s], psi, s, s+1);
+        axpby_ssp_pplus(chi, one, chi, lower[s], psi, s, s-1);
+      }
+      if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, shift_coeffs[s], psi, s, Ls-1); }
+      else{ axpby_ssp_pminus(chi, one, chi, shift_coeffs[s], psi, s, 0); }
+    }
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5Ddag(const FermionField& psi, const FermionField& phi,
+    FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+  {
+    Coeff_t one(1.0);
+    int Ls = this->Ls;
+    for(int s=0; s<Ls; s++){
+      if(s==0) {
+        axpby_ssp_pplus (chi, diag[s], phi, upper[s], psi, s, s+1);
+        axpby_ssp_pminus(chi, one, chi, lower[s], psi, s, Ls-1);
+      } else if (s==(Ls-1)) {
+        axpby_ssp_pplus (chi, diag[s], phi, upper[s], psi, s, 0);
+        axpby_ssp_pminus(chi, one, chi, lower[s], psi, s, s-1);
+      } else {
+        axpby_ssp_pplus (chi, diag[s], phi, upper[s], psi, s, s+1);
+        axpby_ssp_pminus(chi, one, chi, lower[s], psi, s, s-1);
+      }
+    }
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5Ddag_shift(const FermionField& psi, const FermionField& phi,
+    FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper,
+    std::vector<Coeff_t>& shift_coeffs)
+  {
+    Coeff_t one(1.0);
+    int Ls = this->Ls;
+    for(int s=0; s<Ls; s++){
+      if(s==0) {
+        axpby_ssp_pplus (chi, diag[s], phi, upper[s], psi, s, s+1);
+        axpby_ssp_pminus(chi, one, chi, lower[s], psi, s, Ls-1);
+      } else if (s==(Ls-1)) {
+        axpby_ssp_pplus (chi, diag[s], phi, upper[s], psi, s, 0);
+        axpby_ssp_pminus(chi, one, chi, lower[s], psi, s, s-1);
+      } else {
+        axpby_ssp_pplus (chi, diag[s], phi, upper[s], psi, s, s+1);
+        axpby_ssp_pminus(chi, one, chi, lower[s], psi, s, s-1);
+      }
+      if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, shift_coeffs[s], psi, Ls-1, s); }
+      else{ axpby_ssp_pminus(chi, one, chi, shift_coeffs[s], psi, 0, s); }
+    }
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInv(const FermionField& psi, FermionField& chi)
+  {
+    if(this->shift != 0.0){ MooeeInv_shift(psi,chi); return; }
+
+    Coeff_t one(1.0);
+    Coeff_t czero(0.0);
+    chi.checkerboard = psi.checkerboard;
+    int Ls = this->Ls;
+
+    // Apply (L^{\prime})^{-1}
+    axpby_ssp(chi, one, psi, czero, psi, 0, 0);      // chi[0]=psi[0]
+    for(int s=1; s<Ls; s++){
+      axpby_ssp_pplus(chi, one, psi, -this->lee[s-1], chi, s, s-1);// recursion Psi[s] -lee P_+ chi[s-1]
+    }
+
+    // L_m^{-1}
+    for(int s=0; s<Ls-1; s++){ // Chi[ee] = 1 - sum[s<Ls-1] -leem[s]P_- chi
+      axpby_ssp_pminus(chi, one, chi, -this->leem[s], chi, Ls-1, s);
+    }
+
+    // U_m^{-1} D^{-1}
+    for(int s=0; s<Ls-1; s++){
+      axpby_ssp_pplus(chi, one/this->dee[s], chi, -this->ueem[s]/this->dee[Ls-1], chi, s, Ls-1);
+    }
+    axpby_ssp(chi, one/this->dee[Ls-1], chi, czero, chi, Ls-1, Ls-1);
+
+    // Apply U^{-1}
+    for(int s=Ls-2; s>=0; s--){
+      axpby_ssp_pminus(chi, one, chi, -this->uee[s], chi, s, s+1);  // chi[Ls]
+    }
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInv_shift(const FermionField& psi, FermionField& chi)
+  {
+    Coeff_t one(1.0);
+    Coeff_t czero(0.0);
+    chi.checkerboard = psi.checkerboard;
+    int Ls = this->Ls;
+
+    FermionField tmp(psi._grid);
+
+    // Apply (L^{\prime})^{-1}
+    axpby_ssp(chi, one, psi, czero, psi, 0, 0);      // chi[0]=psi[0]
+    axpby_ssp(tmp, czero, tmp, this->MooeeInv_shift_lc[0], psi, 0, 0);
+    for(int s=1; s<Ls; s++){
+      axpby_ssp_pplus(chi, one, psi, -this->lee[s-1], chi, s, s-1);// recursion Psi[s] -lee P_+ chi[s-1]
+      axpby_ssp(tmp, one, tmp, this->MooeeInv_shift_lc[s], psi, 0, s);
+    }
+
+    // L_m^{-1}
+    for(int s=0; s<Ls-1; s++){ // Chi[ee] = 1 - sum[s<Ls-1] -leem[s]P_- chi
+      axpby_ssp_pminus(chi, one, chi, -this->leem[s], chi, Ls-1, s);
+    }
+
+    // U_m^{-1} D^{-1}
+    for(int s=0; s<Ls-1; s++){
+      axpby_ssp_pplus(chi, one/this->dee[s], chi, -this->ueem[s]/this->dee[Ls-1], chi, s, Ls-1);
+    }
+    axpby_ssp(chi, one/this->dee[Ls-1], chi, czero, chi, Ls-1, Ls-1);
+
+    // Apply U^{-1} and add shift term
+    if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, this->MooeeInv_shift_norm[Ls-1], tmp, Ls-1, 0); }
+    else{ axpby_ssp_pminus(chi, one, chi, this->MooeeInv_shift_norm[Ls-1], tmp, Ls-1, 0); }
+    for(int s=Ls-2; s>=0; s--){
+      axpby_ssp_pminus(chi, one, chi, -this->uee[s], chi, s, s+1);  // chi[Ls]
+      if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, this->MooeeInv_shift_norm[s], tmp, s, 0); }
+      else{ axpby_ssp_pminus(chi, one, chi, this->MooeeInv_shift_norm[s], tmp, s, 0); }
+    }
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInvDag(const FermionField& psi, FermionField& chi)
+  {
+    if(this->shift != 0.0){ MooeeInvDag_shift(psi,chi); return; }
+
+    Coeff_t one(1.0);
+    Coeff_t czero(0.0);
+    chi.checkerboard = psi.checkerboard;
+    int Ls = this->Ls;
+
+    // Apply (U^{\prime})^{-dagger}
+    axpby_ssp(chi, one, psi, czero, psi, 0, 0);      // chi[0]=psi[0]
+    for(int s=1; s<Ls; s++){
+      axpby_ssp_pminus(chi, one, psi, -conjugate(this->uee[s-1]), chi, s, s-1);
+    }
+
+    // U_m^{-\dagger}
+    for(int s=0; s<Ls-1; s++){
+      axpby_ssp_pplus(chi, one, chi, -conjugate(this->ueem[s]), chi, Ls-1, s);
+    }
+
+    // L_m^{-\dagger} D^{-dagger}
+    for(int s=0; s<Ls-1; s++){
+      axpby_ssp_pminus(chi, one/conjugate(this->dee[s]), chi, -conjugate(this->leem[s]/this->dee[Ls-1]), chi, s, Ls-1);
+    }
+    axpby_ssp(chi, one/conjugate(this->dee[Ls-1]), chi, czero, chi, Ls-1, Ls-1);
+
+    // Apply L^{-dagger}
+    for(int s=Ls-2; s>=0; s--){
+      axpby_ssp_pplus(chi, one, chi, -conjugate(this->lee[s]), chi, s, s+1);  // chi[Ls]
+    }
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInvDag_shift(const FermionField& psi, FermionField& chi)
+  {
+    Coeff_t one(1.0);
+    Coeff_t czero(0.0);
+    chi.checkerboard = psi.checkerboard;
+    int Ls = this->Ls;
+
+    FermionField tmp(psi._grid);
+
+    // Apply (U^{\prime})^{-dagger} and accumulate (MooeeInvDag_shift_lc)_{j} \psi_{j} in tmp[0]
+    axpby_ssp(chi, one, psi, czero, psi, 0, 0);      // chi[0]=psi[0]
+    axpby_ssp(tmp, czero, tmp, this->MooeeInvDag_shift_lc[0], psi, 0, 0);
+    for(int s=1; s<Ls; s++){
+      axpby_ssp_pminus(chi, one, psi, -conjugate(this->uee[s-1]), chi, s, s-1);
+      axpby_ssp(tmp, one, tmp, this->MooeeInvDag_shift_lc[s], psi, 0, s);
+    }
+
+    // U_m^{-\dagger}
+    for(int s=0; s<Ls-1; s++){
+      axpby_ssp_pplus(chi, one, chi, -conjugate(this->ueem[s]), chi, Ls-1, s);
+    }
+
+    // L_m^{-\dagger} D^{-dagger}
+    for(int s=0; s<Ls-1; s++){
+      axpby_ssp_pminus(chi, one/conjugate(this->dee[s]), chi, -conjugate(this->leem[s]/this->dee[Ls-1]), chi, s, Ls-1);
+    }
+    axpby_ssp(chi, one/conjugate(this->dee[Ls-1]), chi, czero, chi, Ls-1, Ls-1);
+
+    // Apply L^{-dagger} and add shift
+    if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, this->MooeeInvDag_shift_norm[Ls-1], tmp, Ls-1, 0); }
+    else{ axpby_ssp_pminus(chi, one, chi, this->MooeeInvDag_shift_norm[Ls-1], tmp, Ls-1, 0); }
+    for(int s=Ls-2; s>=0; s--){
+      axpby_ssp_pplus(chi, one, chi, -conjugate(this->lee[s]), chi, s, s+1);  // chi[Ls]
+      if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, this->MooeeInvDag_shift_norm[s], tmp, s, 0); }
+      else{ axpby_ssp_pminus(chi, one, chi, this->MooeeInvDag_shift_norm[s], tmp, s, 0); }
+    }
+  }
+
+  #ifdef MOBIUS_EOFA_DPERP_LINALG
+
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplD);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplD);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplD);
+
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(WilsonImplDF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(GparityWilsonImplDF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZWilsonImplDF);
+
+  #endif
+
+}}
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermionvec.cc b/lib/qcd/action/fermion/MobiusEOFAFermionvec.cc
new file mode 100644
index 00000000..59544e5a
--- /dev/null
+++ b/lib/qcd/action/fermion/MobiusEOFAFermionvec.cc
@@ -0,0 +1,654 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./lib/qcd/action/fermion/MobiusEOFAFermionvec.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <pabobyle@ph.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: Peter Boyle <peterboyle@Peters-MacBook-Pro-2.local>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/qcd/action/fermion/FermionCore.h>
+#include <Grid/qcd/action/fermion/MobiusEOFAFermion.h>
+
+namespace Grid {
+namespace QCD {
+
+  /*
+  * Dense matrix versions of routines
+  */
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInv(const FermionField& psi, FermionField& chi)
+  {
+    this->MooeeInternal(psi, chi, DaggerNo, InverseYes);
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInv_shift(const FermionField& psi, FermionField& chi)
+  {
+    this->MooeeInternal(psi, chi, DaggerNo, InverseYes);
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInvDag(const FermionField& psi, FermionField& chi)
+  {
+    this->MooeeInternal(psi, chi, DaggerYes, InverseYes);
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInvDag_shift(const FermionField& psi, FermionField& chi)
+  {
+    this->MooeeInternal(psi, chi, DaggerYes, InverseYes);
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5D(const FermionField& psi, const FermionField& phi,
+    FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+  {
+    GridBase* grid  = psi._grid;
+    int Ls          = this->Ls;
+    int LLs         = grid->_rdimensions[0];
+    const int nsimd = Simd::Nsimd();
+
+    Vector<iSinglet<Simd>> u(LLs);
+    Vector<iSinglet<Simd>> l(LLs);
+    Vector<iSinglet<Simd>> d(LLs);
+
+    assert(Ls/LLs == nsimd);
+    assert(phi.checkerboard == psi.checkerboard);
+
+    chi.checkerboard = psi.checkerboard;
+
+    // just directly address via type pun
+    typedef typename Simd::scalar_type scalar_type;
+    scalar_type* u_p = (scalar_type*) &u[0];
+    scalar_type* l_p = (scalar_type*) &l[0];
+    scalar_type* d_p = (scalar_type*) &d[0];
+
+    for(int o=0; o<LLs; o++){ // outer
+    for(int i=0; i<nsimd; i++){ //inner
+      int s   = o + i*LLs;
+      int ss  = o*nsimd + i;
+      u_p[ss] = upper[s];
+      l_p[ss] = lower[s];
+      d_p[ss] = diag[s];
+    }}
+
+    this->M5Dcalls++;
+    this->M5Dtime -= usecond();
+
+    assert(Nc == 3);
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=LLs){ // adds LLs
+
+      #if 0
+
+        alignas(64) SiteHalfSpinor hp;
+        alignas(64) SiteHalfSpinor hm;
+        alignas(64) SiteSpinor fp;
+        alignas(64) SiteSpinor fm;
+
+        for(int v=0; v<LLs; v++){
+
+          int vp = (v+1)%LLs;
+          int vm = (v+LLs-1)%LLs;
+
+          spProj5m(hp, psi[ss+vp]);
+          spProj5p(hm, psi[ss+vm]);
+
+          if (vp <= v){ rotate(hp, hp, 1); }
+          if (vm >= v){ rotate(hm, hm, nsimd-1); }
+
+          hp = 0.5*hp;
+          hm = 0.5*hm;
+
+          spRecon5m(fp, hp);
+          spRecon5p(fm, hm);
+
+          chi[ss+v] = d[v]*phi[ss+v];
+          chi[ss+v] = chi[ss+v] + u[v]*fp;
+          chi[ss+v] = chi[ss+v] + l[v]*fm;
+
+        }
+
+      #else
+
+        for(int v=0; v<LLs; v++){
+
+          vprefetch(psi[ss+v+LLs]);
+
+          int vp = (v == LLs-1) ? 0     : v+1;
+          int vm = (v == 0)     ? LLs-1 : v-1;
+
+          Simd hp_00 = psi[ss+vp]()(2)(0);
+          Simd hp_01 = psi[ss+vp]()(2)(1);
+          Simd hp_02 = psi[ss+vp]()(2)(2);
+          Simd hp_10 = psi[ss+vp]()(3)(0);
+          Simd hp_11 = psi[ss+vp]()(3)(1);
+          Simd hp_12 = psi[ss+vp]()(3)(2);
+
+          Simd hm_00 = psi[ss+vm]()(0)(0);
+          Simd hm_01 = psi[ss+vm]()(0)(1);
+          Simd hm_02 = psi[ss+vm]()(0)(2);
+          Simd hm_10 = psi[ss+vm]()(1)(0);
+          Simd hm_11 = psi[ss+vm]()(1)(1);
+          Simd hm_12 = psi[ss+vm]()(1)(2);
+
+          if(vp <= v){
+            hp_00.v = Optimization::Rotate::tRotate<2>(hp_00.v);
+            hp_01.v = Optimization::Rotate::tRotate<2>(hp_01.v);
+            hp_02.v = Optimization::Rotate::tRotate<2>(hp_02.v);
+            hp_10.v = Optimization::Rotate::tRotate<2>(hp_10.v);
+            hp_11.v = Optimization::Rotate::tRotate<2>(hp_11.v);
+            hp_12.v = Optimization::Rotate::tRotate<2>(hp_12.v);
+          }
+
+          if(vm >= v){
+            hm_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_00.v);
+            hm_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_01.v);
+            hm_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_02.v);
+            hm_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_10.v);
+            hm_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_11.v);
+            hm_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_12.v);
+          }
+
+          // Can force these to real arithmetic and save 2x.
+          Simd p_00 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_00);
+          Simd p_01 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_01);
+          Simd p_02 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_02);
+          Simd p_10 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_10);
+          Simd p_11 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_11);
+          Simd p_12 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_12);
+          Simd p_20 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_00);
+          Simd p_21 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_01);
+          Simd p_22 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_02);
+          Simd p_30 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_10);
+          Simd p_31 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_11);
+          Simd p_32 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_12);
+
+          vstream(chi[ss+v]()(0)(0), p_00);
+          vstream(chi[ss+v]()(0)(1), p_01);
+          vstream(chi[ss+v]()(0)(2), p_02);
+          vstream(chi[ss+v]()(1)(0), p_10);
+          vstream(chi[ss+v]()(1)(1), p_11);
+          vstream(chi[ss+v]()(1)(2), p_12);
+          vstream(chi[ss+v]()(2)(0), p_20);
+          vstream(chi[ss+v]()(2)(1), p_21);
+          vstream(chi[ss+v]()(2)(2), p_22);
+          vstream(chi[ss+v]()(3)(0), p_30);
+          vstream(chi[ss+v]()(3)(1), p_31);
+          vstream(chi[ss+v]()(3)(2), p_32);
+        }
+
+      #endif
+    }
+
+    this->M5Dtime += usecond();
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5D_shift(const FermionField& psi, const FermionField& phi,
+    FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper,
+    std::vector<Coeff_t>& shift_coeffs)
+  {
+    this->M5D(psi, phi, chi, lower, diag, upper);
+
+    // FIXME: possible gain from vectorizing shift operation as well?
+    Coeff_t one(1.0);
+    int Ls = this->Ls;
+    for(int s=0; s<Ls; s++){
+      if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, shift_coeffs[s], psi, s, Ls-1); }
+      else{ axpby_ssp_pminus(chi, one, chi, shift_coeffs[s], psi, s, 0); }
+    }
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5Ddag(const FermionField& psi, const FermionField& phi,
+    FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper)
+  {
+    GridBase* grid = psi._grid;
+    int Ls  = this->Ls;
+    int LLs = grid->_rdimensions[0];
+    int nsimd = Simd::Nsimd();
+
+    Vector<iSinglet<Simd> > u(LLs);
+    Vector<iSinglet<Simd> > l(LLs);
+    Vector<iSinglet<Simd> > d(LLs);
+
+    assert(Ls/LLs == nsimd);
+    assert(phi.checkerboard == psi.checkerboard);
+
+    chi.checkerboard = psi.checkerboard;
+
+    // just directly address via type pun
+    typedef typename Simd::scalar_type scalar_type;
+    scalar_type* u_p = (scalar_type*) &u[0];
+    scalar_type* l_p = (scalar_type*) &l[0];
+    scalar_type* d_p = (scalar_type*) &d[0];
+
+    for(int o=0; o<LLs; o++){ // outer
+    for(int i=0; i<nsimd; i++){ //inner
+      int s  = o + i*LLs;
+      int ss = o*nsimd + i;
+      u_p[ss] = upper[s];
+      l_p[ss] = lower[s];
+      d_p[ss] = diag[s];
+    }}
+
+    this->M5Dcalls++;
+    this->M5Dtime -= usecond();
+
+    parallel_for(int ss=0; ss<grid->oSites(); ss+=LLs){ // adds LLs
+
+      #if 0
+
+        alignas(64) SiteHalfSpinor hp;
+        alignas(64) SiteHalfSpinor hm;
+        alignas(64) SiteSpinor fp;
+        alignas(64) SiteSpinor fm;
+
+        for(int v=0; v<LLs; v++){
+
+          int vp = (v+1)%LLs;
+          int vm = (v+LLs-1)%LLs;
+
+          spProj5p(hp, psi[ss+vp]);
+          spProj5m(hm, psi[ss+vm]);
+
+          if(vp <= v){ rotate(hp, hp, 1); }
+          if(vm >= v){ rotate(hm, hm, nsimd-1); }
+
+          hp = hp*0.5;
+          hm = hm*0.5;
+          spRecon5p(fp, hp);
+          spRecon5m(fm, hm);
+
+          chi[ss+v] = d[v]*phi[ss+v]+u[v]*fp;
+          chi[ss+v] = chi[ss+v]     +l[v]*fm;
+
+        }
+
+      #else
+
+        for(int v=0; v<LLs; v++){
+
+          vprefetch(psi[ss+v+LLs]);
+
+          int vp = (v == LLs-1) ? 0     : v+1;
+          int vm = (v == 0    ) ? LLs-1 : v-1;
+
+          Simd hp_00 = psi[ss+vp]()(0)(0);
+          Simd hp_01 = psi[ss+vp]()(0)(1);
+          Simd hp_02 = psi[ss+vp]()(0)(2);
+          Simd hp_10 = psi[ss+vp]()(1)(0);
+          Simd hp_11 = psi[ss+vp]()(1)(1);
+          Simd hp_12 = psi[ss+vp]()(1)(2);
+
+          Simd hm_00 = psi[ss+vm]()(2)(0);
+          Simd hm_01 = psi[ss+vm]()(2)(1);
+          Simd hm_02 = psi[ss+vm]()(2)(2);
+          Simd hm_10 = psi[ss+vm]()(3)(0);
+          Simd hm_11 = psi[ss+vm]()(3)(1);
+          Simd hm_12 = psi[ss+vm]()(3)(2);
+
+          if (vp <= v){
+            hp_00.v = Optimization::Rotate::tRotate<2>(hp_00.v);
+            hp_01.v = Optimization::Rotate::tRotate<2>(hp_01.v);
+            hp_02.v = Optimization::Rotate::tRotate<2>(hp_02.v);
+            hp_10.v = Optimization::Rotate::tRotate<2>(hp_10.v);
+            hp_11.v = Optimization::Rotate::tRotate<2>(hp_11.v);
+            hp_12.v = Optimization::Rotate::tRotate<2>(hp_12.v);
+          }
+
+          if(vm >= v){
+            hm_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_00.v);
+            hm_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_01.v);
+            hm_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_02.v);
+            hm_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_10.v);
+            hm_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_11.v);
+            hm_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_12.v);
+          }
+
+          Simd p_00 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_00);
+          Simd p_01 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_01);
+          Simd p_02 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_02);
+          Simd p_10 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_10);
+          Simd p_11 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_11);
+          Simd p_12 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_12);
+          Simd p_20 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_00);
+          Simd p_21 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_01);
+          Simd p_22 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_02);
+          Simd p_30 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_10);
+          Simd p_31 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_11);
+          Simd p_32 = switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_12);
+
+          vstream(chi[ss+v]()(0)(0), p_00);
+          vstream(chi[ss+v]()(0)(1), p_01);
+          vstream(chi[ss+v]()(0)(2), p_02);
+          vstream(chi[ss+v]()(1)(0), p_10);
+          vstream(chi[ss+v]()(1)(1), p_11);
+          vstream(chi[ss+v]()(1)(2), p_12);
+          vstream(chi[ss+v]()(2)(0), p_20);
+          vstream(chi[ss+v]()(2)(1), p_21);
+          vstream(chi[ss+v]()(2)(2), p_22);
+          vstream(chi[ss+v]()(3)(0), p_30);
+          vstream(chi[ss+v]()(3)(1), p_31);
+          vstream(chi[ss+v]()(3)(2), p_32);
+
+        }
+
+      #endif
+
+    }
+
+    this->M5Dtime += usecond();
+  }
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::M5Ddag_shift(const FermionField& psi, const FermionField& phi,
+    FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper,
+    std::vector<Coeff_t>& shift_coeffs)
+  {
+    this->M5Ddag(psi, phi, chi, lower, diag, upper);
+
+    // FIXME: possible gain from vectorizing shift operation as well?
+    Coeff_t one(1.0);
+    int Ls = this->Ls;
+    for(int s=0; s<Ls; s++){
+      if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, shift_coeffs[s], psi, Ls-1, s); }
+      else{ axpby_ssp_pminus(chi, one, chi, shift_coeffs[s], psi, 0, s); }
+    }
+  }
+
+  #ifdef AVX512
+    #include<simd/Intel512common.h>
+    #include<simd/Intel512avx.h>
+    #include<simd/Intel512single.h>
+  #endif
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInternalAsm(const FermionField& psi, FermionField& chi,
+    int LLs, int site, Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm)
+  {
+    #ifndef AVX512
+      {
+        SiteHalfSpinor BcastP;
+        SiteHalfSpinor BcastM;
+        SiteHalfSpinor SiteChiP;
+        SiteHalfSpinor SiteChiM;
+
+        // Ls*Ls * 2 * 12 * vol flops
+        for(int s1=0; s1<LLs; s1++){
+
+          for(int s2=0; s2<LLs; s2++){
+          for(int l=0; l < Simd::Nsimd(); l++){ // simd lane
+
+            int s = s2 + l*LLs;
+            int lex = s2 + LLs*site;
+
+            if( s2==0 && l==0 ){
+              SiteChiP=zero;
+              SiteChiM=zero;
+            }
+
+            for(int sp=0; sp<2;  sp++){
+            for(int co=0; co<Nc; co++){
+              vbroadcast(BcastP()(sp)(co), psi[lex]()(sp)(co), l);
+            }}
+
+            for(int sp=0; sp<2;  sp++){
+            for(int co=0; co<Nc; co++){
+              vbroadcast(BcastM()(sp)(co), psi[lex]()(sp+2)(co), l);
+            }}
+
+            for(int sp=0; sp<2;  sp++){
+            for(int co=0; co<Nc; co++){
+              SiteChiP()(sp)(co) = real_madd(Matp[LLs*s+s1]()()(), BcastP()(sp)(co), SiteChiP()(sp)(co)); // 1100 us.
+              SiteChiM()(sp)(co) = real_madd(Matm[LLs*s+s1]()()(), BcastM()(sp)(co), SiteChiM()(sp)(co)); // each found by commenting out
+            }}
+          }}
+
+          {
+            int lex = s1 + LLs*site;
+            for(int sp=0; sp<2;  sp++){
+            for(int co=0; co<Nc; co++){
+              vstream(chi[lex]()(sp)(co),   SiteChiP()(sp)(co));
+              vstream(chi[lex]()(sp+2)(co), SiteChiM()(sp)(co));
+            }}
+          }
+        }
+      }
+    #else
+      {
+        // pointers
+        //  MASK_REGS;
+        #define Chi_00 %%zmm1
+        #define Chi_01 %%zmm2
+        #define Chi_02 %%zmm3
+        #define Chi_10 %%zmm4
+        #define Chi_11 %%zmm5
+        #define Chi_12 %%zmm6
+        #define Chi_20 %%zmm7
+        #define Chi_21 %%zmm8
+        #define Chi_22 %%zmm9
+        #define Chi_30 %%zmm10
+        #define Chi_31 %%zmm11
+        #define Chi_32 %%zmm12
+
+        #define BCAST0  %%zmm13
+        #define BCAST1  %%zmm14
+        #define BCAST2  %%zmm15
+        #define BCAST3  %%zmm16
+        #define BCAST4  %%zmm17
+        #define BCAST5  %%zmm18
+        #define BCAST6  %%zmm19
+        #define BCAST7  %%zmm20
+        #define BCAST8  %%zmm21
+        #define BCAST9  %%zmm22
+        #define BCAST10 %%zmm23
+        #define BCAST11 %%zmm24
+
+        int incr = LLs*LLs*sizeof(iSinglet<Simd>);
+
+        for(int s1=0; s1<LLs; s1++){
+
+          for(int s2=0; s2<LLs; s2++){
+
+            int lex = s2 + LLs*site;
+            uint64_t a0 = (uint64_t) &Matp[LLs*s2+s1]; // should be cacheable
+            uint64_t a1 = (uint64_t) &Matm[LLs*s2+s1];
+            uint64_t a2 = (uint64_t) &psi[lex];
+
+            for(int l=0; l<Simd::Nsimd(); l++){ // simd lane
+
+              if((s2+l)==0) {
+                asm(
+                      VPREFETCH1(0,%2)              VPREFETCH1(0,%1)
+                      VPREFETCH1(12,%2)  	          VPREFETCH1(13,%2)
+                      VPREFETCH1(14,%2)  	          VPREFETCH1(15,%2)
+                      VBCASTCDUP(0,%2,BCAST0)
+                      VBCASTCDUP(1,%2,BCAST1)
+                      VBCASTCDUP(2,%2,BCAST2)
+                      VBCASTCDUP(3,%2,BCAST3)
+                      VBCASTCDUP(4,%2,BCAST4)       VMULMEM(0,%0,BCAST0,Chi_00)
+                      VBCASTCDUP(5,%2,BCAST5)       VMULMEM(0,%0,BCAST1,Chi_01)
+                      VBCASTCDUP(6,%2,BCAST6)       VMULMEM(0,%0,BCAST2,Chi_02)
+                      VBCASTCDUP(7,%2,BCAST7)       VMULMEM(0,%0,BCAST3,Chi_10)
+                      VBCASTCDUP(8,%2,BCAST8)       VMULMEM(0,%0,BCAST4,Chi_11)
+                      VBCASTCDUP(9,%2,BCAST9)       VMULMEM(0,%0,BCAST5,Chi_12)
+                      VBCASTCDUP(10,%2,BCAST10)     VMULMEM(0,%1,BCAST6,Chi_20)
+                      VBCASTCDUP(11,%2,BCAST11)     VMULMEM(0,%1,BCAST7,Chi_21)
+                      VMULMEM(0,%1,BCAST8,Chi_22)
+                      VMULMEM(0,%1,BCAST9,Chi_30)
+                      VMULMEM(0,%1,BCAST10,Chi_31)
+                      VMULMEM(0,%1,BCAST11,Chi_32)
+                      : : "r" (a0), "r" (a1), "r" (a2)                            );
+              } else {
+                asm(
+                      VBCASTCDUP(0,%2,BCAST0)   VMADDMEM(0,%0,BCAST0,Chi_00)
+                      VBCASTCDUP(1,%2,BCAST1)   VMADDMEM(0,%0,BCAST1,Chi_01)
+                      VBCASTCDUP(2,%2,BCAST2)   VMADDMEM(0,%0,BCAST2,Chi_02)
+                      VBCASTCDUP(3,%2,BCAST3)   VMADDMEM(0,%0,BCAST3,Chi_10)
+                      VBCASTCDUP(4,%2,BCAST4)   VMADDMEM(0,%0,BCAST4,Chi_11)
+                      VBCASTCDUP(5,%2,BCAST5)   VMADDMEM(0,%0,BCAST5,Chi_12)
+                      VBCASTCDUP(6,%2,BCAST6)   VMADDMEM(0,%1,BCAST6,Chi_20)
+                      VBCASTCDUP(7,%2,BCAST7)   VMADDMEM(0,%1,BCAST7,Chi_21)
+                      VBCASTCDUP(8,%2,BCAST8)   VMADDMEM(0,%1,BCAST8,Chi_22)
+                      VBCASTCDUP(9,%2,BCAST9)   VMADDMEM(0,%1,BCAST9,Chi_30)
+                      VBCASTCDUP(10,%2,BCAST10) VMADDMEM(0,%1,BCAST10,Chi_31)
+                      VBCASTCDUP(11,%2,BCAST11) VMADDMEM(0,%1,BCAST11,Chi_32)
+                      : : "r" (a0), "r" (a1), "r" (a2)                            );
+              }
+
+              a0 = a0 + incr;
+              a1 = a1 + incr;
+              a2 = a2 + sizeof(Simd::scalar_type);
+            }
+          }
+
+          {
+            int lexa = s1+LLs*site;
+            asm (
+               VSTORE(0,%0,Chi_00) VSTORE(1 ,%0,Chi_01)  VSTORE(2 ,%0,Chi_02)
+               VSTORE(3,%0,Chi_10) VSTORE(4 ,%0,Chi_11)  VSTORE(5 ,%0,Chi_12)
+               VSTORE(6,%0,Chi_20) VSTORE(7 ,%0,Chi_21)  VSTORE(8 ,%0,Chi_22)
+               VSTORE(9,%0,Chi_30) VSTORE(10,%0,Chi_31)  VSTORE(11,%0,Chi_32)
+               : : "r" ((uint64_t)&chi[lexa]) : "memory" );
+          }
+        }
+      }
+
+      #undef Chi_00
+      #undef Chi_01
+      #undef Chi_02
+      #undef Chi_10
+      #undef Chi_11
+      #undef Chi_12
+      #undef Chi_20
+      #undef Chi_21
+      #undef Chi_22
+      #undef Chi_30
+      #undef Chi_31
+      #undef Chi_32
+
+      #undef BCAST0
+      #undef BCAST1
+      #undef BCAST2
+      #undef BCAST3
+      #undef BCAST4
+      #undef BCAST5
+      #undef BCAST6
+      #undef BCAST7
+      #undef BCAST8
+      #undef BCAST9
+      #undef BCAST10
+      #undef BCAST11
+
+    #endif
+  };
+
+  // Z-mobius version
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInternalZAsm(const FermionField& psi, FermionField& chi,
+    int LLs, int site, Vector<iSinglet<Simd> >& Matp, Vector<iSinglet<Simd> >& Matm)
+  {
+    std::cout << "Error: zMobius not implemented for EOFA" << std::endl;
+    exit(-1);
+  };
+
+  template<class Impl>
+  void MobiusEOFAFermion<Impl>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv)
+  {
+    int Ls  = this->Ls;
+    int LLs = psi._grid->_rdimensions[0];
+    int vol = psi._grid->oSites()/LLs;
+
+    chi.checkerboard = psi.checkerboard;
+
+    Vector<iSinglet<Simd>>   Matp;
+    Vector<iSinglet<Simd>>   Matm;
+    Vector<iSinglet<Simd>>* _Matp;
+    Vector<iSinglet<Simd>>* _Matm;
+
+    //  MooeeInternalCompute(dag,inv,Matp,Matm);
+    if(inv && dag){
+      _Matp = &this->MatpInvDag;
+      _Matm = &this->MatmInvDag;
+    }
+
+    if(inv && (!dag)){
+      _Matp = &this->MatpInv;
+      _Matm = &this->MatmInv;
+    }
+
+    if(!inv){
+      MooeeInternalCompute(dag, inv, Matp, Matm);
+      _Matp = &Matp;
+      _Matm = &Matm;
+    }
+
+    assert(_Matp->size() == Ls*LLs);
+
+    this->MooeeInvCalls++;
+    this->MooeeInvTime -= usecond();
+
+    if(switcheroo<Coeff_t>::iscomplex()){
+      parallel_for(auto site=0; site<vol; site++){
+        MooeeInternalZAsm(psi, chi, LLs, site, *_Matp, *_Matm);
+      }
+    } else {
+      parallel_for(auto site=0; site<vol; site++){
+        MooeeInternalAsm(psi, chi, LLs, site, *_Matp, *_Matm);
+      }
+    }
+
+    this->MooeeInvTime += usecond();
+  }
+
+  #ifdef MOBIUS_EOFA_DPERP_VEC
+
+    INSTANTIATE_DPERP_MOBIUS_EOFA(DomainWallVec5dImplD);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(DomainWallVec5dImplF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZDomainWallVec5dImplD);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZDomainWallVec5dImplF);
+
+    INSTANTIATE_DPERP_MOBIUS_EOFA(DomainWallVec5dImplDF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(DomainWallVec5dImplFH);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZDomainWallVec5dImplDF);
+    INSTANTIATE_DPERP_MOBIUS_EOFA(ZDomainWallVec5dImplFH);
+
+    template void MobiusEOFAFermion<DomainWallVec5dImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<DomainWallVec5dImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<ZDomainWallVec5dImplF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<ZDomainWallVec5dImplD>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+
+    template void MobiusEOFAFermion<DomainWallVec5dImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<DomainWallVec5dImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<ZDomainWallVec5dImplFH>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+    template void MobiusEOFAFermion<ZDomainWallVec5dImplDF>::MooeeInternal(const FermionField& psi, FermionField& chi, int dag, int inv);
+
+  #endif
+
+}}
diff --git a/tests/core/Test_mobius_eofa_even_odd.cc b/tests/core/Test_mobius_eofa_even_odd.cc
new file mode 100644
index 00000000..fceb58f6
--- /dev/null
+++ b/tests/core/Test_mobius_eofa_even_odd.cc
@@ -0,0 +1,241 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/core/Test_dwf_eofa_even_odd.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+template<class d>
+struct scal {
+    d internal;
+};
+
+Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT
+};
+
+int main (int argc, char ** argv)
+{
+    Grid_init(&argc, &argv);
+
+    int threads = GridThread::GetThreads();
+    std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+    const int Ls = 8;
+    // GridCartesian*         UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+    GridCartesian*         UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+    GridCartesian*         FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+    GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+    GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+    std::vector<int> seeds4({1,2,3,4});
+    std::vector<int> seeds5({5,6,7,8});
+
+    GridParallelRNG RNG4(UGrid); RNG4.SeedFixedIntegers(seeds4);
+    GridParallelRNG RNG5(FGrid); RNG5.SeedFixedIntegers(seeds5);
+
+    LatticeFermion    src   (FGrid); random(RNG5, src);
+    LatticeFermion    phi   (FGrid); random(RNG5, phi);
+    LatticeFermion    chi   (FGrid); random(RNG5, chi);
+    LatticeFermion    result(FGrid); result = zero;
+    LatticeFermion    ref   (FGrid); ref = zero;
+    LatticeFermion    tmp   (FGrid); tmp = zero;
+    LatticeFermion    err   (FGrid); err = zero;
+    LatticeGaugeField Umu   (UGrid); SU3::HotConfiguration(RNG4, Umu);
+    std::vector<LatticeColourMatrix> U(4,UGrid);
+
+    // Only one non-zero (y)
+    Umu = zero;
+    for(int nn=0; nn<Nd; nn++){
+        random(RNG4, U[nn]);
+        if(nn>0){ U[nn] = zero; }
+        PokeIndex<LorentzIndex>(Umu, U[nn], nn);
+    }
+
+    RealD b     = 2.5;
+    RealD c     = 1.5;
+    RealD mq1   = 0.1;
+    RealD mq2   = 0.5;
+    RealD mq3   = 1.0;
+    RealD shift = 0.1234;
+    RealD M5    = 1.8;
+    int   pm    = 1;
+    MobiusEOFAFermionR Ddwf(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mq1, mq2, mq3, shift, pm, M5, b, c);
+
+    LatticeFermion src_e (FrbGrid);
+    LatticeFermion src_o (FrbGrid);
+    LatticeFermion r_e   (FrbGrid);
+    LatticeFermion r_o   (FrbGrid);
+    LatticeFermion r_eo  (FGrid);
+    LatticeFermion r_eeoo(FGrid);
+
+    std::cout << GridLogMessage << "==========================================================" << std::endl;
+    std::cout << GridLogMessage << "= Testing that Meo + Moe + Moo + Mee = Munprec " << std::endl;
+    std::cout << GridLogMessage << "==========================================================" << std::endl;
+
+    pickCheckerboard(Even, src_e, src);
+    pickCheckerboard(Odd,  src_o, src);
+
+    Ddwf.Meooe(src_e, r_o); std::cout << GridLogMessage << "Applied Meo" << std::endl;
+    Ddwf.Meooe(src_o, r_e); std::cout << GridLogMessage << "Applied Moe" << std::endl;
+    setCheckerboard(r_eo, r_o);
+    setCheckerboard(r_eo, r_e);
+
+    Ddwf.Mooee(src_e, r_e); std::cout << GridLogMessage << "Applied Mee" << std::endl;
+    Ddwf.Mooee(src_o, r_o); std::cout << GridLogMessage << "Applied Moo" << std::endl;
+    setCheckerboard(r_eeoo, r_e);
+    setCheckerboard(r_eeoo, r_o);
+
+    r_eo = r_eo + r_eeoo;
+    Ddwf.M(src, ref);
+
+    // std::cout << GridLogMessage << r_eo << std::endl;
+    // std::cout << GridLogMessage << ref  << std::endl;
+
+    err = ref - r_eo;
+    std::cout << GridLogMessage << "EO norm diff   " << norm2(err) << " " << norm2(ref) << " " << norm2(r_eo) << std::endl;
+
+    LatticeComplex cerr(FGrid);
+    cerr = localInnerProduct(err,err);
+    // std::cout << GridLogMessage << cerr << std::endl;
+
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+    std::cout << GridLogMessage << "= Test Ddagger is the dagger of D by requiring                " << std::endl;
+    std::cout << GridLogMessage << "=  < phi | Deo | chi > * = < chi | Deo^dag| phi>  " << std::endl;
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+
+    LatticeFermion chi_e (FrbGrid);
+    LatticeFermion chi_o (FrbGrid);
+
+    LatticeFermion dchi_e(FrbGrid);
+    LatticeFermion dchi_o(FrbGrid);
+
+    LatticeFermion phi_e (FrbGrid);
+    LatticeFermion phi_o (FrbGrid);
+
+    LatticeFermion dphi_e(FrbGrid);
+    LatticeFermion dphi_o(FrbGrid);
+
+    pickCheckerboard(Even, chi_e, chi);
+    pickCheckerboard(Odd , chi_o, chi);
+    pickCheckerboard(Even, phi_e, phi);
+    pickCheckerboard(Odd , phi_o, phi);
+
+    Ddwf.Meooe   (chi_e, dchi_o);
+    Ddwf.Meooe   (chi_o, dchi_e);
+    Ddwf.MeooeDag(phi_e, dphi_o);
+    Ddwf.MeooeDag(phi_o, dphi_e);
+
+    ComplexD pDce = innerProduct(phi_e, dchi_e);
+    ComplexD pDco = innerProduct(phi_o, dchi_o);
+    ComplexD cDpe = innerProduct(chi_e, dphi_e);
+    ComplexD cDpo = innerProduct(chi_o, dphi_o);
+
+    std::cout << GridLogMessage << "e " << pDce << " " << cDpe << std::endl;
+    std::cout << GridLogMessage << "o " << pDco << " " << cDpo << std::endl;
+
+    std::cout << GridLogMessage << "pDce - conj(cDpo) " << pDce-conj(cDpo) << std::endl;
+    std::cout << GridLogMessage << "pDco - conj(cDpe) " << pDco-conj(cDpe) << std::endl;
+
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+    std::cout << GridLogMessage << "= Test MeeInv Mee = 1                                         " << std::endl;
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+
+    pickCheckerboard(Even, chi_e, chi);
+    pickCheckerboard(Odd , chi_o, chi);
+
+    Ddwf.Mooee   (chi_e, src_e);
+    Ddwf.MooeeInv(src_e, phi_e);
+
+    Ddwf.Mooee   (chi_o, src_o);
+    Ddwf.MooeeInv(src_o, phi_o);
+
+    setCheckerboard(phi, phi_e);
+    setCheckerboard(phi, phi_o);
+
+    err = phi - chi;
+    std::cout << GridLogMessage << "norm diff   " << norm2(err) << std::endl;
+
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+    std::cout << GridLogMessage << "= Test MeeInvDag MeeDag = 1                                   " << std::endl;
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+
+    pickCheckerboard(Even, chi_e, chi);
+    pickCheckerboard(Odd , chi_o, chi);
+
+    Ddwf.MooeeDag   (chi_e, src_e);
+    Ddwf.MooeeInvDag(src_e, phi_e);
+
+    Ddwf.MooeeDag   (chi_o, src_o);
+    Ddwf.MooeeInvDag(src_o, phi_o);
+
+    setCheckerboard(phi, phi_e);
+    setCheckerboard(phi, phi_o);
+
+    err = phi - chi;
+    std::cout << GridLogMessage << "norm diff   " << norm2(err) << std::endl;
+
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+    std::cout << GridLogMessage << "= Test MpcDagMpc is Hermitian              " << std::endl;
+    std::cout << GridLogMessage << "==============================================================" << std::endl;
+
+    random(RNG5, phi);
+    random(RNG5, chi);
+    pickCheckerboard(Even, chi_e, chi);
+    pickCheckerboard(Odd , chi_o, chi);
+    pickCheckerboard(Even, phi_e, phi);
+    pickCheckerboard(Odd , phi_o, phi);
+    RealD t1,t2;
+
+    SchurDiagMooeeOperator<MobiusEOFAFermionR,LatticeFermion> HermOpEO(Ddwf);
+    HermOpEO.MpcDagMpc(chi_e, dchi_e, t1, t2);
+    HermOpEO.MpcDagMpc(chi_o, dchi_o, t1, t2);
+
+    HermOpEO.MpcDagMpc(phi_e, dphi_e, t1, t2);
+    HermOpEO.MpcDagMpc(phi_o, dphi_o, t1, t2);
+
+    pDce = innerProduct(phi_e, dchi_e);
+    pDco = innerProduct(phi_o, dchi_o);
+    cDpe = innerProduct(chi_e, dphi_e);
+    cDpo = innerProduct(chi_o, dphi_o);
+
+    std::cout << GridLogMessage << "e " << pDce << " " << cDpe << std::endl;
+    std::cout << GridLogMessage << "o " << pDco << " " << cDpo << std::endl;
+
+    std::cout << GridLogMessage << "pDce - conj(cDpo) " << pDco-conj(cDpo) << std::endl;
+    std::cout << GridLogMessage << "pDco - conj(cDpe) " << pDce-conj(cDpe) << std::endl;
+
+    Grid_finalize();
+}
diff --git a/tests/debug/Test_heatbath_mobius_eofa.cc b/tests/debug/Test_heatbath_mobius_eofa.cc
new file mode 100644
index 00000000..a952873d
--- /dev/null
+++ b/tests/debug/Test_heatbath_mobius_eofa.cc
@@ -0,0 +1,104 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/debug/Test_heatbath_dwf_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// This program sets up the initial pseudofermion field |Phi> = Meofa^{-1/2}*|eta>, and
+// then uses this Phi to compute the action <Phi|Meofa|Phi>.
+// If all is working, one should find that <eta|eta> = <Phi|Meofa|Phi>.
+//////////////////////////////////////////////////////////////////////////////////////////
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+// Parameters for test
+const std::vector<int> grid_dim = { 8, 8, 8, 8 };
+const int              Ls       = 8;
+const int              Npoles   = 12;
+const RealD            b        = 2.5;
+const RealD            c        = 1.5;
+const RealD            mf       = 0.01;
+const RealD            mpv      = 1.0;
+const RealD            M5       = 1.8;
+
+int main(int argc, char** argv)
+{
+  Grid_init(&argc, &argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is set up to use " << threads << " threads" << std::endl;
+
+  // Initialize spacetime grid
+  std::cout << GridLogMessage << "Lattice dimensions: " << grid_dim << "  Ls: " << Ls << std::endl;
+  GridCartesian*         UGrid   = SpaceTimeGrid::makeFourDimGrid(grid_dim,
+                                    GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian*         FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Set up RNGs
+  std::vector<int> seeds4({1, 2, 3, 4});
+  std::vector<int> seeds5({5, 6, 7, 8});
+  GridParallelRNG RNG5(FGrid);
+  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);
+  RNG4.SeedFixedIntegers(seeds4);
+
+  // Random gauge field
+  LatticeGaugeField Umu(UGrid);
+  SU3::HotConfiguration(RNG4, Umu);
+
+  MobiusEOFAFermionR Lop(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf,  mf, mpv,  0.0, -1, M5, b, c);
+  MobiusEOFAFermionR Rop(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mpv, mf, mpv, -1.0,  1, M5, b, c);
+
+  // Construct the action and test the heatbath (zero initial guess)
+  {
+    OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, Npoles);
+    ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
+    ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, false);
+
+    Meofa.refresh(Umu, RNG5);
+    printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
+  }
+
+  // Construct the action and test the heatbath (forecasted initial guesses)
+  {
+    OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, Npoles);
+    ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
+    ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, true);
+
+    Meofa.refresh(Umu, RNG5);
+    printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
+  }
+
+  return 0;
+}
diff --git a/tests/debug/Test_heatbath_mobius_eofa_gparity.cc b/tests/debug/Test_heatbath_mobius_eofa_gparity.cc
new file mode 100644
index 00000000..08c6d566
--- /dev/null
+++ b/tests/debug/Test_heatbath_mobius_eofa_gparity.cc
@@ -0,0 +1,109 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/debug/Test_heatbath_dwf_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// This program sets up the initial pseudofermion field |Phi> = Meofa^{-1/2}*|eta>, and
+// then uses this Phi to compute the action <Phi|Meofa|Phi>.
+// If all is working, one should find that <eta|eta> = <Phi|Meofa|Phi>.
+//////////////////////////////////////////////////////////////////////////////////////////
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+typedef GparityWilsonImplR FermionImplPolicy;
+typedef GparityMobiusEOFAFermionR FermionAction;
+typedef typename FermionAction::FermionField FermionField;
+
+// Parameters for test
+const std::vector<int> grid_dim = { 8, 8, 8, 8 };
+const int              Ls       = 8;
+const int              Npoles   = 12;
+const RealD            b        = 2.5;
+const RealD            c        = 1.5;
+const RealD            mf       = 0.01;
+const RealD            mpv      = 1.0;
+const RealD            M5       = 1.8;
+
+int main(int argc, char** argv)
+{
+  Grid_init(&argc, &argv);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is set up to use " << threads << " threads" << std::endl;
+
+  // Initialize spacetime grid
+  std::cout << GridLogMessage << "Lattice dimensions: " << grid_dim << "  Ls: " << Ls << std::endl;
+  GridCartesian*         UGrid   = SpaceTimeGrid::makeFourDimGrid(grid_dim,
+                                    GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian*         FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Set up RNGs
+  std::vector<int> seeds4({1, 2, 3, 4});
+  std::vector<int> seeds5({5, 6, 7, 8});
+  GridParallelRNG RNG5(FGrid);
+  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);
+  RNG4.SeedFixedIntegers(seeds4);
+
+  // Random gauge field
+  LatticeGaugeField Umu(UGrid);
+  SU3::HotConfiguration(RNG4, Umu);
+
+  FermionAction::ImplParams params;
+  FermionAction Lop(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf,  mf, mpv,  0.0, -1, M5, b, c, params);
+  FermionAction Rop(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mpv, mf, mpv, -1.0,  1, M5, b, c, params);
+
+  // Construct the action and test the heatbath (zero initial guess)
+  {
+    OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, Npoles);
+    ConjugateGradient<FermionField> CG(1.0e-12, 5000);
+    ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, false);
+
+    Meofa.refresh(Umu, RNG5);
+    printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
+  }
+
+  // Construct the action and test the heatbath (forecasted initial guesses)
+  {
+    OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, Npoles);
+    ConjugateGradient<FermionField> CG(1.0e-12, 5000);
+    ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, true);
+
+    Meofa.refresh(Umu, RNG5);
+    printf("<Phi|Meofa|Phi> = %1.15e\n", Meofa.S(Umu));
+  }
+
+  return 0;
+}
diff --git a/tests/debug/Test_reweight_dwf_eofa_gparity.cc b/tests/debug/Test_reweight_dwf_eofa_gparity.cc
index b77ec33e..bb0fd98e 100644
--- a/tests/debug/Test_reweight_dwf_eofa_gparity.cc
+++ b/tests/debug/Test_reweight_dwf_eofa_gparity.cc
@@ -2,7 +2,7 @@
 
 Grid physics library, www.github.com/paboyle/Grid
 
-Source file: ./tests/debug/Test_reweight_dwf_eofa.cc
+Source file: ./tests/debug/Test_reweight_dwf_eofa_gparity.cc
 
 Copyright (C) 2017
 
diff --git a/tests/debug/Test_reweight_mobius_eofa.cc b/tests/debug/Test_reweight_mobius_eofa.cc
new file mode 100644
index 00000000..c4fa78d0
--- /dev/null
+++ b/tests/debug/Test_reweight_mobius_eofa.cc
@@ -0,0 +1,215 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/debug/Test_reweight_dwf_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+// parameters for test
+const std::vector<int> grid_dim = { 8, 8, 8, 8 };
+const int Ls = 8;
+const int Nhits = 10;
+const int max_iter = 5000;
+const RealD b  = 2.5;
+const RealD c  = 1.5;
+const RealD mf = 0.1;
+const RealD mb = 0.11;
+const RealD M5 = 1.8;
+const RealD stop_tol = 1.0e-12;
+
+RealD mean(const std::vector<RealD>& data)
+{
+    int N = data.size();
+    RealD mean(0.0);
+    for(int i=0; i<N; ++i){ mean += data[i]; }
+    return mean/RealD(N);
+}
+
+RealD jack_mean(const std::vector<RealD>& data, int sample)
+{
+    int N = data.size();
+    RealD mean(0.0);
+    for(int i=0; i<N; ++i){ if(i != sample){ mean += data[i]; } }
+    return mean/RealD(N-1);
+}
+
+RealD jack_std(const std::vector<RealD>& jacks, RealD mean)
+{
+    int N = jacks.size();
+    RealD std(0.0);
+    for(int i=0; i<N; ++i){ std += std::pow(jacks[i]-mean, 2.0); }
+    return std::sqrt(RealD(N-1)/RealD(N)*std);
+}
+
+std::vector<RealD> jack_stats(const std::vector<RealD>& data)
+{
+    int N = data.size();
+    std::vector<RealD> jack_samples(N);
+    std::vector<RealD> jack_stats(2);
+
+    jack_stats[0] = mean(data);
+    for(int i=0; i<N; i++){ jack_samples[i] = jack_mean(data,i); }
+    jack_stats[1] = jack_std(jack_samples, jack_stats[0]);
+    return jack_stats;
+}
+
+int main(int argc, char **argv)
+{
+  Grid_init(&argc, &argv);
+
+  // Initialize spacetime grid
+  std::cout << GridLogMessage << "Lattice dimensions: "
+    << grid_dim << "   Ls: " << Ls << std::endl;
+  GridCartesian* UGrid = SpaceTimeGrid::makeFourDimGrid(grid_dim,
+      GridDefaultSimd(Nd, vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian* FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Set up RNGs
+  std::vector<int> seeds4({1, 2, 3, 4});
+  std::vector<int> seeds5({5, 6, 7, 8});
+  GridParallelRNG RNG5(FGrid);
+  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);
+  RNG4.SeedFixedIntegers(seeds4);
+
+  // Random gauge field
+  LatticeGaugeField Umu(UGrid);
+  SU3::HotConfiguration(RNG4, Umu);
+
+  // Initialize RHMC fermion operators
+  MobiusFermionR Ddwf_f(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, M5, b, c);
+  MobiusFermionR Ddwf_b(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, M5, b, c);
+  SchurDiagMooeeOperator<MobiusFermionR, LatticeFermion> MdagM(Ddwf_f);
+  SchurDiagMooeeOperator<MobiusFermionR, LatticeFermion> VdagV(Ddwf_b);
+
+  // Degree 12 rational approximations to x^(1/4) and x^(-1/4)
+  double     lo = 0.0001;
+  double     hi = 95.0;
+  int precision = 64;
+  int    degree = 12;
+  AlgRemez remez(lo, hi, precision);
+  std::cout << GridLogMessage << "Generating degree " << degree << " for x^(1/4)" << std::endl;
+  remez.generateApprox(degree, 1, 4);
+  MultiShiftFunction PowerQuarter(remez, stop_tol, false);
+  MultiShiftFunction PowerNegQuarter(remez, stop_tol, true);
+
+  // Stochastically estimate reweighting factor via RHMC
+  RealD scale = std::sqrt(0.5);
+  std::vector<RealD> rw_rhmc(Nhits);
+  ConjugateGradientMultiShift<LatticeFermion> msCG_V(max_iter, PowerQuarter);
+  ConjugateGradientMultiShift<LatticeFermion> msCG_M(max_iter, PowerNegQuarter);
+  std::cout.precision(12);
+
+  for(int hit=0; hit<Nhits; hit++){
+
+    // Gaussian source
+    LatticeFermion Phi    (Ddwf_f.FermionGrid());
+    LatticeFermion PhiOdd (Ddwf_f.FermionRedBlackGrid());
+    std::vector<LatticeFermion> tmp(2, Ddwf_f.FermionRedBlackGrid());
+    gaussian(RNG5, Phi);
+    Phi = Phi*scale;
+
+    pickCheckerboard(Odd, PhiOdd, Phi);
+
+    // evaluate -log(rw)
+    msCG_V(VdagV, PhiOdd, tmp[0]);
+    msCG_M(MdagM, tmp[0], tmp[1]);
+    rw_rhmc[hit] = norm2(tmp[1]) - norm2(PhiOdd);
+    std::cout << std::endl << "==================================================" << std::endl;
+    std::cout << " --- RHMC: Hit " << hit << ": rw = " << rw_rhmc[hit];
+    std::cout << std::endl << "==================================================" << std::endl << std::endl;
+
+  }
+
+  // Initialize EOFA fermion operators
+  RealD shift_L = 0.0;
+  RealD shift_R = -1.0;
+  int pm = 1;
+  MobiusEOFAFermionR Deofa_L(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, mf, mb, shift_L, pm, M5, b, c);
+  MobiusEOFAFermionR Deofa_R(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, mf, mb, shift_R, pm, M5, b, c);
+  MdagMLinearOperator<MobiusEOFAFermionR, LatticeFermion> LdagL(Deofa_L);
+  MdagMLinearOperator<MobiusEOFAFermionR, LatticeFermion> RdagR(Deofa_R);
+
+  // Stochastically estimate reweighting factor via EOFA
+  RealD k = Deofa_L.k;
+  std::vector<RealD> rw_eofa(Nhits);
+  ConjugateGradient<LatticeFermion> CG(stop_tol, max_iter);
+  SchurRedBlackDiagMooeeSolve<LatticeFermion> SchurSolver(CG);
+
+  // Compute -log(Z), where: ( RHMC det ratio ) = Z * ( EOFA det ratio )
+  RealD Z = std::pow(b+c+1.0,Ls) + mf*std::pow(b+c-1.0,Ls);
+  Z /= std::pow(b+c+1.0,Ls) + mb*std::pow(b+c-1.0,Ls);
+  Z = -12.0*grid_dim[0]*grid_dim[1]*grid_dim[2]*grid_dim[3]*std::log(Z);
+
+  for(int hit=0; hit<Nhits; hit++){
+
+    // Gaussian source
+    LatticeFermion Phi       (Deofa_L.FermionGrid());
+    LatticeFermion spProj_Phi(Deofa_L.FermionGrid());
+    std::vector<LatticeFermion> tmp(2, Deofa_L.FermionGrid());
+    gaussian(RNG5, Phi);
+    Phi = Phi*scale;
+
+    // evaluate -log(rw)
+    // LH term
+    for(int s=0; s<Ls; ++s){ axpby_ssp_pminus(spProj_Phi, 0.0, Phi, 1.0, Phi, s, s); }
+    Deofa_L.Omega(spProj_Phi, tmp[0], -1, 0);
+    G5R5(tmp[1], tmp[0]);
+    tmp[0] = zero;
+    SchurSolver(Deofa_L, tmp[1], tmp[0]);
+    Deofa_L.Dtilde(tmp[0], tmp[1]);
+    Deofa_L.Omega(tmp[1], tmp[0], -1, 1);
+    rw_eofa[hit] = Z - k*innerProduct(spProj_Phi,tmp[0]).real();
+
+    // RH term
+    for(int s=0; s<Ls; ++s){ axpby_ssp_pplus(spProj_Phi, 0.0, Phi, 1.0, Phi, s, s); }
+    Deofa_R.Omega(spProj_Phi, tmp[0], 1, 0);
+    G5R5(tmp[1], tmp[0]);
+    tmp[0] = zero;
+    SchurSolver(Deofa_R, tmp[1], tmp[0]);
+    Deofa_R.Dtilde(tmp[0], tmp[1]);
+    Deofa_R.Omega(tmp[1], tmp[0], 1, 1);
+    rw_eofa[hit] += k*innerProduct(spProj_Phi,tmp[0]).real();
+    std::cout << std::endl << "==================================================" << std::endl;
+    std::cout << " --- EOFA: Hit " << hit << ": rw = " << rw_eofa[hit];
+    std::cout << std::endl << "==================================================" << std::endl << std::endl;
+
+  }
+
+  std::vector<RealD> rhmc_result = jack_stats(rw_rhmc);
+  std::vector<RealD> eofa_result = jack_stats(rw_eofa);
+  std::cout << std::endl << "RHMC: rw = " << rhmc_result[0] << " +/- " << rhmc_result[1] << std::endl;
+  std::cout << std::endl << "EOFA: rw = " << eofa_result[0] << " +/- " << eofa_result[1] << std::endl;
+
+  Grid_finalize();
+}
diff --git a/tests/debug/Test_reweight_mobius_eofa_gparity.cc b/tests/debug/Test_reweight_mobius_eofa_gparity.cc
new file mode 100644
index 00000000..11a242d2
--- /dev/null
+++ b/tests/debug/Test_reweight_mobius_eofa_gparity.cc
@@ -0,0 +1,218 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/debug/Test_reweight_dwf_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+typedef typename GparityDomainWallFermionR::FermionField FermionField;
+
+// parameters for test
+const std::vector<int> grid_dim = { 8, 8, 8, 8 };
+const int Ls = 8;
+const int Nhits = 10;
+const int max_iter = 5000;
+const RealD b  = 2.5;
+const RealD c  = 1.5;
+const RealD mf = 0.1;
+const RealD mb = 0.11;
+const RealD M5 = 1.8;
+const RealD stop_tol = 1.0e-12;
+
+RealD mean(const std::vector<RealD>& data)
+{
+    int N = data.size();
+    RealD mean(0.0);
+    for(int i=0; i<N; ++i){ mean += data[i]; }
+    return mean/RealD(N);
+}
+
+RealD jack_mean(const std::vector<RealD>& data, int sample)
+{
+    int N = data.size();
+    RealD mean(0.0);
+    for(int i=0; i<N; ++i){ if(i != sample){ mean += data[i]; } }
+    return mean/RealD(N-1);
+}
+
+RealD jack_std(const std::vector<RealD>& jacks, RealD mean)
+{
+    int N = jacks.size();
+    RealD std(0.0);
+    for(int i=0; i<N; ++i){ std += std::pow(jacks[i]-mean, 2.0); }
+    return std::sqrt(RealD(N-1)/RealD(N)*std);
+}
+
+std::vector<RealD> jack_stats(const std::vector<RealD>& data)
+{
+    int N = data.size();
+    std::vector<RealD> jack_samples(N);
+    std::vector<RealD> jack_stats(2);
+
+    jack_stats[0] = mean(data);
+    for(int i=0; i<N; i++){ jack_samples[i] = jack_mean(data,i); }
+    jack_stats[1] = jack_std(jack_samples, jack_stats[0]);
+    return jack_stats;
+}
+
+int main(int argc, char **argv)
+{
+  Grid_init(&argc, &argv);
+
+  // Initialize spacetime grid
+  std::cout << GridLogMessage << "Lattice dimensions: "
+    << grid_dim << "   Ls: " << Ls << std::endl;
+  GridCartesian* UGrid = SpaceTimeGrid::makeFourDimGrid(grid_dim,
+      GridDefaultSimd(Nd, vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian* UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian* FGrid = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian* FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Set up RNGs
+  std::vector<int> seeds4({1, 2, 3, 4});
+  std::vector<int> seeds5({5, 6, 7, 8});
+  GridParallelRNG RNG5(FGrid);
+  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);
+  RNG4.SeedFixedIntegers(seeds4);
+
+  // Random gauge field
+  LatticeGaugeField Umu(UGrid);
+  SU3::HotConfiguration(RNG4, Umu);
+
+  // Initialize RHMC fermion operators
+  GparityDomainWallFermionR::ImplParams params;
+  GparityMobiusFermionR Ddwf_f(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, M5, b, c, params);
+  GparityMobiusFermionR Ddwf_b(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, M5, b, c, params);
+  SchurDiagMooeeOperator<GparityMobiusFermionR, FermionField> MdagM(Ddwf_f);
+  SchurDiagMooeeOperator<GparityMobiusFermionR, FermionField> VdagV(Ddwf_b);
+
+  // Degree 12 rational approximations to x^(1/4) and x^(-1/4)
+  double     lo = 0.0001;
+  double     hi = 95.0;
+  int precision = 64;
+  int    degree = 12;
+  AlgRemez remez(lo, hi, precision);
+  std::cout << GridLogMessage << "Generating degree " << degree << " for x^(1/4)" << std::endl;
+  remez.generateApprox(degree, 1, 4);
+  MultiShiftFunction PowerQuarter(remez, stop_tol, false);
+  MultiShiftFunction PowerNegQuarter(remez, stop_tol, true);
+
+  // Stochastically estimate reweighting factor via RHMC
+  RealD scale = std::sqrt(0.5);
+  std::vector<RealD> rw_rhmc(Nhits);
+  ConjugateGradientMultiShift<FermionField> msCG_V(max_iter, PowerQuarter);
+  ConjugateGradientMultiShift<FermionField> msCG_M(max_iter, PowerNegQuarter);
+  std::cout.precision(12);
+
+  for(int hit=0; hit<Nhits; hit++){
+
+    // Gaussian source
+    FermionField Phi    (Ddwf_f.FermionGrid());
+    FermionField PhiOdd (Ddwf_f.FermionRedBlackGrid());
+    std::vector<FermionField> tmp(2, Ddwf_f.FermionRedBlackGrid());
+    gaussian(RNG5, Phi);
+    Phi = Phi*scale;
+
+    pickCheckerboard(Odd, PhiOdd, Phi);
+
+    // evaluate -log(rw)
+    msCG_V(VdagV, PhiOdd, tmp[0]);
+    msCG_M(MdagM, tmp[0], tmp[1]);
+    rw_rhmc[hit] = norm2(tmp[1]) - norm2(PhiOdd);
+    std::cout << std::endl << "==================================================" << std::endl;
+    std::cout << " --- RHMC: Hit " << hit << ": rw = " << rw_rhmc[hit];
+    std::cout << std::endl << "==================================================" << std::endl << std::endl;
+
+  }
+
+  // Initialize EOFA fermion operators
+  RealD shift_L = 0.0;
+  RealD shift_R = -1.0;
+  int pm = 1;
+  GparityMobiusEOFAFermionR Deofa_L(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, mf, mb, shift_L, pm, M5, b, c, params);
+  GparityMobiusEOFAFermionR Deofa_R(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, mf, mb, shift_R, pm, M5, b, c, params);
+  MdagMLinearOperator<GparityMobiusEOFAFermionR, FermionField> LdagL(Deofa_L);
+  MdagMLinearOperator<GparityMobiusEOFAFermionR, FermionField> RdagR(Deofa_R);
+
+  // Stochastically estimate reweighting factor via EOFA
+  RealD k = Deofa_L.k;
+  std::vector<RealD> rw_eofa(Nhits);
+  ConjugateGradient<FermionField> CG(stop_tol, max_iter);
+  SchurRedBlackDiagMooeeSolve<FermionField> SchurSolver(CG);
+
+  // Compute -log(Z), where: ( RHMC det ratio ) = Z * ( EOFA det ratio )
+  RealD Z = std::pow(b+c+1.0,Ls) + mf*std::pow(b+c-1.0,Ls);
+  Z /= std::pow(b+c+1.0,Ls) + mb*std::pow(b+c-1.0,Ls);
+  Z = -12.0*grid_dim[0]*grid_dim[1]*grid_dim[2]*grid_dim[3]*std::log(Z);
+
+  for(int hit=0; hit<Nhits; hit++){
+
+    // Gaussian source
+    FermionField Phi       (Deofa_L.FermionGrid());
+    FermionField spProj_Phi(Deofa_L.FermionGrid());
+    std::vector<FermionField> tmp(2, Deofa_L.FermionGrid());
+    gaussian(RNG5, Phi);
+    Phi = Phi*scale;
+
+    // evaluate -log(rw)
+    // LH term
+    for(int s=0; s<Ls; ++s){ axpby_ssp_pminus(spProj_Phi, 0.0, Phi, 1.0, Phi, s, s); }
+    Deofa_L.Omega(spProj_Phi, tmp[0], -1, 0);
+    G5R5(tmp[1], tmp[0]);
+    tmp[0] = zero;
+    SchurSolver(Deofa_L, tmp[1], tmp[0]);
+    Deofa_L.Dtilde(tmp[0], tmp[1]);
+    Deofa_L.Omega(tmp[1], tmp[0], -1, 1);
+    rw_eofa[hit] = 2.0*Z - k*innerProduct(spProj_Phi,tmp[0]).real();
+
+    // RH term
+    for(int s=0; s<Ls; ++s){ axpby_ssp_pplus(spProj_Phi, 0.0, Phi, 1.0, Phi, s, s); }
+    Deofa_R.Omega(spProj_Phi, tmp[0], 1, 0);
+    G5R5(tmp[1], tmp[0]);
+    tmp[0] = zero;
+    SchurSolver(Deofa_R, tmp[1], tmp[0]);
+    Deofa_R.Dtilde(tmp[0], tmp[1]);
+    Deofa_R.Omega(tmp[1], tmp[0], 1, 1);
+    rw_eofa[hit] += k*innerProduct(spProj_Phi,tmp[0]).real();
+    std::cout << std::endl << "==================================================" << std::endl;
+    std::cout << " --- EOFA: Hit " << hit << ": rw = " << rw_eofa[hit];
+    std::cout << std::endl << "==================================================" << std::endl << std::endl;
+
+  }
+
+  std::vector<RealD> rhmc_result = jack_stats(rw_rhmc);
+  std::vector<RealD> eofa_result = jack_stats(rw_eofa);
+  std::cout << std::endl << "RHMC: rw = " << rhmc_result[0] << " +/- " << rhmc_result[1] << std::endl;
+  std::cout << std::endl << "EOFA: rw = " << eofa_result[0] << " +/- " << eofa_result[1] << std::endl;
+
+  Grid_finalize();
+}
diff --git a/tests/forces/Test_dwf_gpforce_eofa.cc b/tests/forces/Test_dwf_gpforce_eofa.cc
index 50789c89..3afeaa43 100644
--- a/tests/forces/Test_dwf_gpforce_eofa.cc
+++ b/tests/forces/Test_dwf_gpforce_eofa.cc
@@ -71,9 +71,9 @@ int main (int argc, char** argv)
   int threads = GridThread::GetThreads();
   std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
 
-  LatticeFermion phi        (FGrid);  gaussian(RNG5, phi);
-  LatticeFermion Mphi       (FGrid);
-  LatticeFermion MphiPrime  (FGrid);
+  FermionField phi        (FGrid);  gaussian(RNG5, phi);
+  FermionField Mphi       (FGrid);
+  FermionField MphiPrime  (FGrid);
 
   LatticeGaugeField U(UGrid);
   SU3::HotConfiguration(RNG4,U);
diff --git a/tests/forces/Test_mobius_force_eofa.cc b/tests/forces/Test_mobius_force_eofa.cc
new file mode 100644
index 00000000..2a5a7d04
--- /dev/null
+++ b/tests/forces/Test_mobius_force_eofa.cc
@@ -0,0 +1,166 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/forces/Test_dwf_force_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+int main (int argc, char** argv)
+{
+  Grid_init(&argc, &argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+
+  const int Ls = 8;
+
+  GridCartesian         *UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         *FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Want a different conf at every run
+  // First create an instance of an engine.
+  std::random_device rnd_device;
+  // Specify the engine and distribution.
+  std::mt19937 mersenne_engine(rnd_device());
+  std::uniform_int_distribution<int> dist(1, 100);
+
+  auto gen = std::bind(dist, mersenne_engine);
+  std::vector<int> seeds4(4);
+  generate(begin(seeds4), end(seeds4), gen);
+
+  //std::vector<int> seeds4({1,2,3,5});
+  std::vector<int> seeds5({5,6,7,8});
+  GridParallelRNG RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+  LatticeFermion phi        (FGrid);  gaussian(RNG5, phi);
+  LatticeFermion Mphi       (FGrid);
+  LatticeFermion MphiPrime  (FGrid);
+
+  LatticeGaugeField U(UGrid);
+  SU3::HotConfiguration(RNG4,U);
+
+  ////////////////////////////////////
+  // Unmodified matrix element
+  ////////////////////////////////////
+  RealD b  = 2.5;
+  RealD c  = 1.5;
+  RealD mf = 0.01;
+  RealD mb = 1.0;
+  RealD M5 = 1.8;
+  MobiusEOFAFermionR Lop(U, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, mf, mb, 0.0, -1, M5, b, c);
+  MobiusEOFAFermionR Rop(U, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, mf, mb, -1.0, 1, M5, b, c);
+  OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, 12);
+  ConjugateGradient<LatticeFermion> CG(1.0e-12, 5000);
+  ExactOneFlavourRatioPseudoFermionAction<WilsonImplR> Meofa(Lop, Rop, CG, Params, false);
+
+  Meofa.refresh(U, RNG5);
+  RealD S = Meofa.S(U); // pdag M p
+
+  // get the deriv of phidag M phi with respect to "U"
+  LatticeGaugeField UdSdU(UGrid);
+  Meofa.deriv(U, UdSdU);
+
+  ////////////////////////////////////
+  // Modify the gauge field a little
+  ////////////////////////////////////
+  RealD dt = 0.0001;
+
+  LatticeColourMatrix mommu(UGrid);
+  LatticeColourMatrix forcemu(UGrid);
+  LatticeGaugeField mom(UGrid);
+  LatticeGaugeField Uprime(UGrid);
+
+  for(int mu=0; mu<Nd; mu++){
+
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+
+    PokeIndex<LorentzIndex>(mom, mommu, mu);
+
+    // fourth order exponential approx
+    parallel_for(auto i=mom.begin(); i<mom.end(); i++){
+      Uprime[i](mu) = U[i](mu) + mom[i](mu)*U[i](mu)*dt + mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt/2.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt/6.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt/24.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt/120.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt*dt/720.0);
+    }
+  }
+
+  /*Ddwf.ImportGauge(Uprime);
+  Ddwf.M          (phi,MphiPrime);
+
+  ComplexD Sprime    = innerProduct(MphiPrime   ,MphiPrime);*/
+  RealD Sprime = Meofa.S(Uprime);
+
+  //////////////////////////////////////////////
+  // Use derivative to estimate dS
+  //////////////////////////////////////////////
+
+  LatticeComplex dS(UGrid);
+  dS = zero;
+  for(int mu=0; mu<Nd; mu++){
+    mommu = PeekIndex<LorentzIndex>(UdSdU, mu);
+    mommu = Ta(mommu)*2.0;
+    PokeIndex<LorentzIndex>(UdSdU, mommu, mu);
+  }
+
+  for(int mu=0; mu<Nd; mu++){
+    forcemu = PeekIndex<LorentzIndex>(UdSdU, mu);
+    mommu   = PeekIndex<LorentzIndex>(mom, mu);
+
+    // Update PF action density
+    dS = dS + trace(mommu*forcemu)*dt;
+  }
+
+  ComplexD dSpred = sum(dS);
+
+  /*std::cout << GridLogMessage << " S      " << S << std::endl;
+  std::cout << GridLogMessage << " Sprime " << Sprime << std::endl;
+  std::cout << GridLogMessage << "dS      " << Sprime-S << std::endl;
+  std::cout << GridLogMessage << "predict dS    " << dSpred << std::endl;*/
+  printf("\nS = %1.15e\n", S);
+  printf("Sprime = %1.15e\n", Sprime);
+  printf("dS = %1.15e\n", Sprime - S);
+  printf("real(dS_predict) = %1.15e\n", dSpred.real());
+  printf("imag(dS_predict) = %1.15e\n\n", dSpred.imag());
+
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
+
+  std::cout << GridLogMessage << "Done" << std::endl;
+  Grid_finalize();
+}
diff --git a/tests/forces/Test_mobius_gpforce_eofa.cc b/tests/forces/Test_mobius_gpforce_eofa.cc
new file mode 100644
index 00000000..72f1dee2
--- /dev/null
+++ b/tests/forces/Test_mobius_gpforce_eofa.cc
@@ -0,0 +1,171 @@
+/*************************************************************************************
+
+Grid physics library, www.github.com/paboyle/Grid
+
+Source file: ./tests/forces/Test_dwf_force_eofa.cc
+
+Copyright (C) 2017
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: David Murphy <dmurphy@phys.columbia.edu>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+See the full license in the file "LICENSE" in the top level distribution directory
+*************************************************************************************/
+/*  END LEGAL */
+
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+typedef GparityWilsonImplR FermionImplPolicy;
+typedef GparityMobiusEOFAFermionR FermionAction;
+typedef typename FermionAction::FermionField FermionField;
+
+int main (int argc, char** argv)
+{
+  Grid_init(&argc, &argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+
+  const int Ls = 8;
+
+  GridCartesian         *UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()), GridDefaultMpi());
+  GridRedBlackCartesian *UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         *FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls, UGrid);
+  GridRedBlackCartesian *FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls, UGrid);
+
+  // Want a different conf at every run
+  // First create an instance of an engine.
+  std::random_device rnd_device;
+  // Specify the engine and distribution.
+  std::mt19937 mersenne_engine(rnd_device());
+  std::uniform_int_distribution<int> dist(1, 100);
+
+  auto gen = std::bind(dist, mersenne_engine);
+  std::vector<int> seeds4(4);
+  generate(begin(seeds4), end(seeds4), gen);
+
+  //std::vector<int> seeds4({1,2,3,5});
+  std::vector<int> seeds5({5,6,7,8});
+  GridParallelRNG RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+  GridParallelRNG RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+
+  int threads = GridThread::GetThreads();
+  std::cout << GridLogMessage << "Grid is setup to use " << threads << " threads" << std::endl;
+
+  FermionField phi        (FGrid);  gaussian(RNG5, phi);
+  FermionField Mphi       (FGrid);
+  FermionField MphiPrime  (FGrid);
+
+  LatticeGaugeField U(UGrid);
+  SU3::HotConfiguration(RNG4,U);
+
+  ////////////////////////////////////
+  // Unmodified matrix element
+  ////////////////////////////////////
+  RealD b  = 2.5;
+  RealD c  = 1.5;
+  RealD mf = 0.01;
+  RealD mb = 1.0;
+  RealD M5 = 1.8;
+  FermionAction::ImplParams params;
+  FermionAction Lop(U, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mf, mf, mb, 0.0, -1, M5, b, c, params);
+  FermionAction Rop(U, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mb, mf, mb, -1.0, 1, M5, b, c, params);
+  OneFlavourRationalParams Params(0.95, 100.0, 5000, 1.0e-12, 12);
+  ConjugateGradient<FermionField> CG(1.0e-12, 5000);
+  ExactOneFlavourRatioPseudoFermionAction<FermionImplPolicy> Meofa(Lop, Rop, CG, Params, false);
+
+  Meofa.refresh(U, RNG5);
+  RealD S = Meofa.S(U); // pdag M p
+
+  // get the deriv of phidag M phi with respect to "U"
+  LatticeGaugeField UdSdU(UGrid);
+  Meofa.deriv(U, UdSdU);
+
+  ////////////////////////////////////
+  // Modify the gauge field a little
+  ////////////////////////////////////
+  RealD dt = 0.0001;
+
+  LatticeColourMatrix mommu(UGrid);
+  LatticeColourMatrix forcemu(UGrid);
+  LatticeGaugeField mom(UGrid);
+  LatticeGaugeField Uprime(UGrid);
+
+  for(int mu=0; mu<Nd; mu++){
+
+    SU3::GaussianFundamentalLieAlgebraMatrix(RNG4, mommu); // Traceless antihermitian momentum; gaussian in lie alg
+
+    PokeIndex<LorentzIndex>(mom, mommu, mu);
+
+    // fourth order exponential approx
+    parallel_for(auto i=mom.begin(); i<mom.end(); i++){
+      Uprime[i](mu) = U[i](mu) + mom[i](mu)*U[i](mu)*dt + mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt/2.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt/6.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt/24.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt/120.0)
+                        + mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *mom[i](mu) *U[i](mu)*(dt*dt*dt*dt*dt*dt/720.0);
+    }
+  }
+
+  /*Ddwf.ImportGauge(Uprime);
+  Ddwf.M          (phi,MphiPrime);
+
+  ComplexD Sprime    = innerProduct(MphiPrime   ,MphiPrime);*/
+  RealD Sprime = Meofa.S(Uprime);
+
+  //////////////////////////////////////////////
+  // Use derivative to estimate dS
+  //////////////////////////////////////////////
+
+  LatticeComplex dS(UGrid);
+  dS = zero;
+  for(int mu=0; mu<Nd; mu++){
+    mommu = PeekIndex<LorentzIndex>(UdSdU, mu);
+    mommu = Ta(mommu)*2.0;
+    PokeIndex<LorentzIndex>(UdSdU, mommu, mu);
+  }
+
+  for(int mu=0; mu<Nd; mu++){
+    forcemu = PeekIndex<LorentzIndex>(UdSdU, mu);
+    mommu   = PeekIndex<LorentzIndex>(mom, mu);
+
+    // Update PF action density
+    dS = dS + trace(mommu*forcemu)*dt;
+  }
+
+  ComplexD dSpred = sum(dS);
+
+  /*std::cout << GridLogMessage << " S      " << S << std::endl;
+  std::cout << GridLogMessage << " Sprime " << Sprime << std::endl;
+  std::cout << GridLogMessage << "dS      " << Sprime-S << std::endl;
+  std::cout << GridLogMessage << "predict dS    " << dSpred << std::endl;*/
+  printf("\nS = %1.15e\n", S);
+  printf("Sprime = %1.15e\n", Sprime);
+  printf("dS = %1.15e\n", Sprime - S);
+  printf("real(dS_predict) = %1.15e\n", dSpred.real());
+  printf("imag(dS_predict) = %1.15e\n\n", dSpred.imag());
+
+  assert( fabs(real(Sprime-S-dSpred)) < 1.0 ) ;
+
+  std::cout << GridLogMessage << "Done" << std::endl;
+  Grid_finalize();
+}

From d9cd4f027336b650f7da0add0f3f380611cda7db Mon Sep 17 00:00:00 2001
From: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Wed, 23 Aug 2017 15:07:18 +0100
Subject: [PATCH 059/115] Staggered multinode block cg debugged. Missing global
 sum. Code stalls and resumes on KNL at cambridge. Curious.

CG iterations 23ms each, then 3200 ms pauses. Mean bandwidth reports
as 200MB/s. Comms dominant in the report. However, the time behaviour suggests it
is *bursty*.... Could be swap to disk?
---
 .../iterative/BlockConjugateGradient.h        |  9 ++-
 lib/lattice/Lattice_reduction.h               | 38 ++++++++----
 .../fermion/ImprovedStaggeredFermion5D.cc     | 60 +++++++++++++++++++
 .../fermion/ImprovedStaggeredFermion5D.h      | 10 ++++
 .../solver/Test_staggered_block_cg_unprec.cc  |  8 ++-
 5 files changed, 110 insertions(+), 15 deletions(-)

diff --git a/lib/algorithms/iterative/BlockConjugateGradient.h b/lib/algorithms/iterative/BlockConjugateGradient.h
index 9418f63c..d7817c05 100644
--- a/lib/algorithms/iterative/BlockConjugateGradient.h
+++ b/lib/algorithms/iterative/BlockConjugateGradient.h
@@ -199,7 +199,12 @@ void BlockCGrQsolve(LinearOperatorBase<Field> &Linop, const Field &B, Field &X)
 
   Linop.HermOp(X, AD);
   tmp = B - AD;  
+  //std::cout << GridLogMessage << " initial tmp " << norm2(tmp)<< std::endl;
   ThinQRfact (m_rr, m_C, m_Cinv, Q, tmp);
+  //std::cout << GridLogMessage << " initial Q " << norm2(Q)<< std::endl;
+  //std::cout << GridLogMessage << " m_rr " << m_rr<<std::endl;
+  //std::cout << GridLogMessage << " m_C " << m_C<<std::endl;
+  //std::cout << GridLogMessage << " m_Cinv " << m_Cinv<<std::endl;
   D=Q;
 
   std::cout << GridLogMessage<<"BlockCGrQ computed initial residual and QR fact " <<std::endl;
@@ -221,13 +226,15 @@ void BlockCGrQsolve(LinearOperatorBase<Field> &Linop, const Field &B, Field &X)
     MatrixTimer.Start();
     Linop.HermOp(D, Z);      
     MatrixTimer.Stop();
+    //std::cout << GridLogMessage << " norm2 Z " <<norm2(Z)<<std::endl;
 
     //4. M  = [D^dag Z]^{-1}
     sliceInnerTimer.Start();
     sliceInnerProductMatrix(m_DZ,D,Z,Orthog);
     sliceInnerTimer.Stop();
     m_M       = m_DZ.inverse();
-
+    //std::cout << GridLogMessage << " m_DZ " <<m_DZ<<std::endl;
+    
     //5. X  = X + D MC
     m_tmp     = m_M * m_C;
     sliceMaddTimer.Start();
diff --git a/lib/lattice/Lattice_reduction.h b/lib/lattice/Lattice_reduction.h
index 38982891..db012c8c 100644
--- a/lib/lattice/Lattice_reduction.h
+++ b/lib/lattice/Lattice_reduction.h
@@ -369,6 +369,7 @@ static void sliceMaddVector(Lattice<vobj> &R,std::vector<RealD> &a,const Lattice
   }
 };
 
+/*
 inline GridBase         *makeSubSliceGrid(const GridBase *BlockSolverGrid,int Orthog)
 {
   int NN    = BlockSolverGrid->_ndimension;
@@ -387,6 +388,7 @@ inline GridBase         *makeSubSliceGrid(const GridBase *BlockSolverGrid,int Or
   }
   return (GridBase *)new GridCartesian(latt_phys,simd_phys,mpi_phys); 
 }
+*/
 
 template<class vobj>
 static void sliceMaddMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice<vobj> &X,const Lattice<vobj> &Y,int Orthog,RealD scale=1.0) 
@@ -398,14 +400,15 @@ static void sliceMaddMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice
   int Nblock = X._grid->GlobalDimensions()[Orthog];
 
   GridBase *FullGrid  = X._grid;
-  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
+  //  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
 
-  Lattice<vobj> Xslice(SliceGrid);
-  Lattice<vobj> Rslice(SliceGrid);
+  //  Lattice<vobj> Xslice(SliceGrid);
+  //  Lattice<vobj> Rslice(SliceGrid);
 
   assert( FullGrid->_simd_layout[Orthog]==1);
   int nh =  FullGrid->_ndimension;
-  int nl = SliceGrid->_ndimension;
+  //  int nl = SliceGrid->_ndimension;
+  int nl = nh-1;
 
   //FIXME package in a convenient iterator
   //Should loop over a plane orthogonal to direction "Orthog"
@@ -448,14 +451,14 @@ static void sliceMulMatrix (Lattice<vobj> &R,Eigen::MatrixXcd &aa,const Lattice<
   int Nblock = X._grid->GlobalDimensions()[Orthog];
 
   GridBase *FullGrid  = X._grid;
-  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
-
-  Lattice<vobj> Xslice(SliceGrid);
-  Lattice<vobj> Rslice(SliceGrid);
+  //  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
+  //  Lattice<vobj> Xslice(SliceGrid);
+  //  Lattice<vobj> Rslice(SliceGrid);
 
   assert( FullGrid->_simd_layout[Orthog]==1);
   int nh =  FullGrid->_ndimension;
-  int nl = SliceGrid->_ndimension;
+  //  int nl = SliceGrid->_ndimension;
+  int nl=1;
 
   //FIXME package in a convenient iterator
   //Should loop over a plane orthogonal to direction "Orthog"
@@ -498,18 +501,19 @@ static void sliceInnerProductMatrix(  Eigen::MatrixXcd &mat, const Lattice<vobj>
   typedef typename vobj::vector_type vector_type;
   
   GridBase *FullGrid  = lhs._grid;
-  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
+  //  GridBase *SliceGrid = makeSubSliceGrid(FullGrid,Orthog);
   
   int Nblock = FullGrid->GlobalDimensions()[Orthog];
   
-  Lattice<vobj> Lslice(SliceGrid);
-  Lattice<vobj> Rslice(SliceGrid);
+  //  Lattice<vobj> Lslice(SliceGrid);
+  //  Lattice<vobj> Rslice(SliceGrid);
   
   mat = Eigen::MatrixXcd::Zero(Nblock,Nblock);
 
   assert( FullGrid->_simd_layout[Orthog]==1);
   int nh =  FullGrid->_ndimension;
-  int nl = SliceGrid->_ndimension;
+  //  int nl = SliceGrid->_ndimension;
+  int nl = nh-1;
 
   //FIXME package in a convenient iterator
   //Should loop over a plane orthogonal to direction "Orthog"
@@ -550,6 +554,14 @@ static void sliceInnerProductMatrix(  Eigen::MatrixXcd &mat, const Lattice<vobj>
       mat += mat_thread;
     }  
   }
+
+  for(int i=0;i<Nblock;i++){
+  for(int j=0;j<Nblock;j++){
+    ComplexD sum = mat(i,j);
+    FullGrid->GlobalSum(sum);
+    mat(i,j)=sum;
+  }}
+
   return;
 }
 
diff --git a/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.cc b/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.cc
index 61a3c559..7d988d89 100644
--- a/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.cc
+++ b/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.cc
@@ -230,8 +230,15 @@ void ImprovedStaggeredFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOr
 {
   Compressor compressor;
   int LLs = in._grid->_rdimensions[0];
+
+
+
+  DhopTotalTime -= usecond();
+  DhopCommTime -= usecond();
   st.HaloExchange(in,compressor);
+  DhopCommTime += usecond();
   
+  DhopComputeTime -= usecond();
   // Dhop takes the 4d grid from U, and makes a 5d index for fermion
   if (dag == DaggerYes) {
     parallel_for (int ss = 0; ss < U._grid->oSites(); ss++) {
@@ -244,12 +251,15 @@ void ImprovedStaggeredFermion5D<Impl>::DhopInternal(StencilImpl & st, LebesgueOr
 	Kernels::DhopSite(st,lo,U,UUU,st.CommBuf(),LLs,sU,in,out);
     }
   }
+  DhopComputeTime += usecond();
+  DhopTotalTime   += usecond();
 }
 
 
 template<class Impl>
 void ImprovedStaggeredFermion5D<Impl>::DhopOE(const FermionField &in, FermionField &out,int dag)
 {
+  DhopCalls+=1;
   conformable(in._grid,FermionRedBlackGrid());    // verifies half grid
   conformable(in._grid,out._grid); // drops the cb check
 
@@ -261,6 +271,7 @@ void ImprovedStaggeredFermion5D<Impl>::DhopOE(const FermionField &in, FermionFie
 template<class Impl>
 void ImprovedStaggeredFermion5D<Impl>::DhopEO(const FermionField &in, FermionField &out,int dag)
 {
+  DhopCalls+=1;
   conformable(in._grid,FermionRedBlackGrid());    // verifies half grid
   conformable(in._grid,out._grid); // drops the cb check
 
@@ -272,6 +283,7 @@ void ImprovedStaggeredFermion5D<Impl>::DhopEO(const FermionField &in, FermionFie
 template<class Impl>
 void ImprovedStaggeredFermion5D<Impl>::Dhop(const FermionField &in, FermionField &out,int dag)
 {
+  DhopCalls+=2;
   conformable(in._grid,FermionGrid()); // verifies full grid
   conformable(in._grid,out._grid);
 
@@ -280,6 +292,54 @@ void ImprovedStaggeredFermion5D<Impl>::Dhop(const FermionField &in, FermionField
   DhopInternal(Stencil,Lebesgue,Umu,UUUmu,in,out,dag);
 }
 
+template<class Impl>
+void ImprovedStaggeredFermion5D<Impl>::Report(void) 
+{
+  std::vector<int> latt = GridDefaultLatt();          
+  RealD volume = Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt[mu];
+  RealD NP = _FourDimGrid->_Nprocessors;
+  RealD NN = _FourDimGrid->NodeCount();
+
+  std::cout << GridLogMessage << "#### Dhop calls report " << std::endl;
+
+  std::cout << GridLogMessage << "ImprovedStaggeredFermion5D Number of DhopEO Calls   : " 
+	    << DhopCalls   << std::endl;
+  std::cout << GridLogMessage << "ImprovedStaggeredFermion5D TotalTime   /Calls       : " 
+	    << DhopTotalTime   / DhopCalls << " us" << std::endl;
+  std::cout << GridLogMessage << "ImprovedStaggeredFermion5D CommTime    /Calls       : " 
+	    << DhopCommTime    / DhopCalls << " us" << std::endl;
+  std::cout << GridLogMessage << "ImprovedStaggeredFermion5D ComputeTime/Calls        : " 
+	    << DhopComputeTime / DhopCalls << " us" << std::endl;
+
+  // Average the compute time
+  _FourDimGrid->GlobalSum(DhopComputeTime);
+  DhopComputeTime/=NP;
+
+  RealD mflops = 1154*volume*DhopCalls/DhopComputeTime/2; // 2 for red black counting
+  std::cout << GridLogMessage << "Average mflops/s per call                : " << mflops << std::endl;
+  std::cout << GridLogMessage << "Average mflops/s per call per rank       : " << mflops/NP << std::endl;
+  std::cout << GridLogMessage << "Average mflops/s per call per node       : " << mflops/NN << std::endl;
+  
+  RealD Fullmflops = 1154*volume*DhopCalls/(DhopTotalTime)/2; // 2 for red black counting
+  std::cout << GridLogMessage << "Average mflops/s per call (full)         : " << Fullmflops << std::endl;
+  std::cout << GridLogMessage << "Average mflops/s per call per rank (full): " << Fullmflops/NP << std::endl;
+  std::cout << GridLogMessage << "Average mflops/s per call per node (full): " << Fullmflops/NN << std::endl;
+
+  std::cout << GridLogMessage << "ImprovedStaggeredFermion5D Stencil"    <<std::endl;  Stencil.Report();
+  std::cout << GridLogMessage << "ImprovedStaggeredFermion5D StencilEven"<<std::endl;  StencilEven.Report();
+  std::cout << GridLogMessage << "ImprovedStaggeredFermion5D StencilOdd" <<std::endl;  StencilOdd.Report();
+}
+template<class Impl>
+void ImprovedStaggeredFermion5D<Impl>::ZeroCounters(void) 
+{
+  DhopCalls       = 0;
+  DhopTotalTime    = 0;
+  DhopCommTime    = 0;
+  DhopComputeTime = 0;
+  Stencil.ZeroCounters();
+  StencilEven.ZeroCounters();
+  StencilOdd.ZeroCounters();
+}
 
 /////////////////////////////////////////////////////////////////////////
 // Implement the general interface. Here we use SAME mass on all slices
diff --git a/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.h b/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.h
index 4961da49..ca1a955a 100644
--- a/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.h
+++ b/lib/qcd/action/fermion/ImprovedStaggeredFermion5D.h
@@ -55,6 +55,16 @@ namespace QCD {
       FermionField _tmp;
       FermionField &tmp(void) { return _tmp; }
 
+      ////////////////////////////////////////
+      // Performance monitoring
+      ////////////////////////////////////////
+      void Report(void);
+      void ZeroCounters(void);
+      double DhopTotalTime;
+      double DhopCalls;
+      double DhopCommTime;
+      double DhopComputeTime;
+
       ///////////////////////////////////////////////////////////////
       // Implement the abstract base
       ///////////////////////////////////////////////////////////////
diff --git a/tests/solver/Test_staggered_block_cg_unprec.cc b/tests/solver/Test_staggered_block_cg_unprec.cc
index 8db41e98..f54bc3b2 100644
--- a/tests/solver/Test_staggered_block_cg_unprec.cc
+++ b/tests/solver/Test_staggered_block_cg_unprec.cc
@@ -75,7 +75,7 @@ int main (int argc, char ** argv)
   LatticeGaugeField Umu(UGrid); SU3::HotConfiguration(pRNG,Umu);
 
   RealD mass=0.003;
-  ImprovedStaggeredFermion5DR Ds(Umu,Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass);
+  ImprovedStaggeredFermion5DR Ds(Umu,Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass); 
   MdagMLinearOperator<ImprovedStaggeredFermion5DR,FermionField> HermOp(Ds);
 
   ConjugateGradient<FermionField> CG(1.0e-8,10000);
@@ -99,21 +99,27 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << " Calling 5d CG for "<<Ls <<" right hand sides" <<std::endl;
   std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
   result=zero;
+  Ds.ZeroCounters();
   CG(HermOp,src,result);
+  Ds.Report();
   std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
 
   std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
   std::cout << GridLogMessage << " Calling multiRHS CG for "<<Ls <<" right hand sides" <<std::endl;
   std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
   result=zero;
+  Ds.ZeroCounters();
   mCG(HermOp,src,result);
+  Ds.Report();
   std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
 
   std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
   std::cout << GridLogMessage << " Calling Block CG for "<<Ls <<" right hand sides" <<std::endl;
   std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
   result=zero;
+  Ds.ZeroCounters();
   BCGrQ(HermOp,src,result);
+  Ds.Report();
   std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
 
 

From b61835c1a53ab6077840aabdb3ef8e77ea161008 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Wed, 23 Aug 2017 12:33:48 -0400
Subject: [PATCH 060/115] Added inplace version of intrinsic G-parity twist to
 hand-unrolled kernel

---
 lib/qcd/action/fermion/WilsonKernelsHand.cc | 90 ++++++++++++++++++++-
 1 file changed, 86 insertions(+), 4 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index 866e30d2..045a2cda 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -47,9 +47,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define LOAD_CHIMU(DIR,F)						\
   { const SiteSpinor & ref (in._odata[offset]); LOAD_CHIMU_BODY(F); }
 
-#define LOAD_CHIMU_GPARITY(DIR,F)					\
-  { int g; const SiteSpinor & ref = GparityGetChi<SiteSpinor>(g,in._odata.data(),DIR,F,SE,st); LOAD_CHIMU_BODY(g); }
-
 #define LOAD_CHI_BODY(F)				\
     Chi_00 = ref(F)(0)(0);\
     Chi_01 = ref(F)(0)(1);\
@@ -61,10 +58,95 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define LOAD_CHI(DIR,F)						\
   {const SiteHalfSpinor &ref(buf[offset]); LOAD_CHI_BODY(F); }
 
-#define LOAD_CHI_GPARITY(DIR,F)						\
+
+//G-parity implementations using implementation method
+#define LOAD_CHIMU_GPARITY_IMPL(DIR,F)					\
+  { int g; const SiteSpinor & ref = GparityGetChi<SiteSpinor>(g,in._odata.data(),DIR,F,SE,st); LOAD_CHIMU_BODY(g); }
+
+#define LOAD_CHI_GPARITY_IMPL(DIR,F)					\
   { int g; const SiteHalfSpinor &ref = GparityGetChi<SiteHalfSpinor>(g,buf,DIR,F,SE,st); LOAD_CHI_BODY(g); }
 
 
+//G-parity implementations using in-place intrinsic ops
+
+//1l 1h -> 1h 1l
+//0l 0h , 1h 1l -> 0l 1h 0h,1l
+//0h,1l -> 1l,0h
+//if( (distance == 1 && !perm_will_occur) || (distance == -1 && perm_will_occur) )
+//Pulled fermion through forwards face, GPBC on upper component
+//Need 0= 0l 1h   1= 1l 0h
+//else if( (distance == -1 && !perm) || (distance == 1 && perm) )
+//Pulled fermion through backwards face, GPBC on lower component
+//Need 0= 1l 0h   1= 0l 1h
+#define DO_TWIST(INTO,S,C,F, tmp1, tmp2, tmp3, tmp4)			\
+  permute(tmp1, ref(1)(S)(C), permute_type);				\
+  exchange(tmp2,tmp3, ref(0)(S)(C), tmp1, permute_type);		\
+  permute(tmp4, tmp3, permute_type);					\
+  if( (distance == 1 && !perm) || (distance == -1 && perm) ){		\
+    INTO = F == 0 ? tmp2 : tmp4;					\
+  }else if( (distance == -1 && !perm) || (distance == 1 && perm) ){	\
+    INTO = F == 0 ? tmp4 : tmp2;					\
+  }
+
+#define LOAD_CHI_SETUP(DIR,F)						\
+  int g = F;								\
+  const int direction = st._directions[DIR];				\
+  const int distance = st._distances[DIR];				\
+  const int sl = st._grid->_simd_layout[direction];			\
+  int inplace_twist = 0;						\
+  if(SE->_around_the_world && this->Params.twists[DIR % 4]){		\
+    if(sl == 1){							\
+      g = (F+1) % 2;							\
+    }else{								\
+      inplace_twist = 1;						\
+    }									\
+  }  
+
+#define LOAD_CHIMU_GPARITY_INPLACE_TWIST(DIR,F)				\
+  { const SiteSpinor &ref(in._odata[offset]);				\
+    LOAD_CHI_SETUP(DIR,F);						\
+    if(!inplace_twist){							\
+      LOAD_CHIMU_BODY(g);						\
+    }else{								\
+      const int permute_type = st._grid->PermuteType(direction);	\
+      DO_TWIST(Chimu_00,0,0,F,  U_00,U_01,U_10,U_11);			\
+      DO_TWIST(Chimu_01,0,1,F,  U_20,U_21,U_00,U_01);			\
+      DO_TWIST(Chimu_02,0,2,F,  U_10,U_11,U_20,U_21);			\
+      DO_TWIST(Chimu_10,1,0,F,  U_00,U_01,U_10,U_11);			\
+      DO_TWIST(Chimu_11,1,1,F,  U_20,U_21,U_00,U_01);			\
+      DO_TWIST(Chimu_12,1,2,F,  U_10,U_11,U_20,U_21);			\
+      DO_TWIST(Chimu_20,2,0,F,  U_00,U_01,U_10,U_11);			\
+      DO_TWIST(Chimu_21,2,1,F,  U_20,U_21,U_00,U_01);			\
+      DO_TWIST(Chimu_22,2,2,F,  U_10,U_11,U_20,U_21);			\
+      DO_TWIST(Chimu_30,3,0,F,  U_00,U_01,U_10,U_11);			\
+      DO_TWIST(Chimu_31,3,1,F,  U_20,U_21,U_00,U_01);			\
+      DO_TWIST(Chimu_32,3,2,F,  U_10,U_11,U_20,U_21);			\
+    } \
+  }
+
+
+#define LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F)				\
+  { const SiteHalfSpinor &ref(buf[offset]);				\
+    LOAD_CHI_SETUP(DIR,F);						\
+    if(!inplace_twist){							\
+      LOAD_CHI_BODY(g);							\
+    }else{								\
+      const int permute_type = st._grid->PermuteType(direction);	\
+      DO_TWIST(Chi_00,0,0,F,  U_00,U_01,U_10,U_11);			\
+      DO_TWIST(Chi_01,0,1,F,  U_20,U_21,UChi_00,UChi_01);		\
+      DO_TWIST(Chi_02,0,2,F,  UChi_02,UChi_10,UChi_11,UChi_12);		\
+      DO_TWIST(Chi_10,1,0,F,  U_00,U_01,U_10,U_11);			\
+      DO_TWIST(Chi_11,1,1,F,  U_20,U_21,UChi_00,UChi_01);		\
+      DO_TWIST(Chi_12,1,2,F,  UChi_02,UChi_10,UChi_11,UChi_12);		\
+    }									\
+  }
+
+//#define LOAD_CHI_GPARITY(DIR,F) LOAD_CHI_GPARITY_IMPL(DIR,F)
+#define LOAD_CHI_GPARITY(DIR,F) LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F)
+
+//#define LOAD_CHIMU_GPARITY(DIR,F) LOAD_CHIMU_GPARITY_IMPL(DIR,F)
+#define LOAD_CHIMU_GPARITY(DIR,F) LOAD_CHIMU_GPARITY_INPLACE_TWIST(DIR,F)
+
 // To splat or not to splat depends on the implementation
 #define MULT_2SPIN_BODY \
   Impl::loadLinkElement(U_00,ref()(0,0));	\

From dd8f1ea189b3c9998b9c94bbe595ce8b89efc2ae Mon Sep 17 00:00:00 2001
From: David Murphy <david.murphy@ligo.org>
Date: Wed, 23 Aug 2017 13:17:26 -0400
Subject: [PATCH 061/115] Vectorized Mobius EOFA Dperp + shift operation

---
 .../action/fermion/MobiusEOFAFermionvec.cc    | 367 +++++++++++++++++-
 1 file changed, 348 insertions(+), 19 deletions(-)

diff --git a/lib/qcd/action/fermion/MobiusEOFAFermionvec.cc b/lib/qcd/action/fermion/MobiusEOFAFermionvec.cc
index 59544e5a..c4eaf0f3 100644
--- a/lib/qcd/action/fermion/MobiusEOFAFermionvec.cc
+++ b/lib/qcd/action/fermion/MobiusEOFAFermionvec.cc
@@ -213,15 +213,180 @@ namespace QCD {
     FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper,
     std::vector<Coeff_t>& shift_coeffs)
   {
-    this->M5D(psi, phi, chi, lower, diag, upper);
+    #if 0
 
-    // FIXME: possible gain from vectorizing shift operation as well?
-    Coeff_t one(1.0);
-    int Ls = this->Ls;
-    for(int s=0; s<Ls; s++){
-      if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, shift_coeffs[s], psi, s, Ls-1); }
-      else{ axpby_ssp_pminus(chi, one, chi, shift_coeffs[s], psi, s, 0); }
-    }
+      this->M5D(psi, phi, chi, lower, diag, upper);
+
+      // FIXME: possible gain from vectorizing shift operation as well?
+      Coeff_t one(1.0);
+      int Ls = this->Ls;
+      for(int s=0; s<Ls; s++){
+        if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, shift_coeffs[s], psi, s, Ls-1); }
+        else{ axpby_ssp_pminus(chi, one, chi, shift_coeffs[s], psi, s, 0); }
+      }
+
+    #else
+
+      GridBase* grid  = psi._grid;
+      int Ls          = this->Ls;
+      int LLs         = grid->_rdimensions[0];
+      const int nsimd = Simd::Nsimd();
+
+      Vector<iSinglet<Simd>> u(LLs);
+      Vector<iSinglet<Simd>> l(LLs);
+      Vector<iSinglet<Simd>> d(LLs);
+      Vector<iSinglet<Simd>> s(LLs);
+
+      assert(Ls/LLs == nsimd);
+      assert(phi.checkerboard == psi.checkerboard);
+
+      chi.checkerboard = psi.checkerboard;
+
+      // just directly address via type pun
+      typedef typename Simd::scalar_type scalar_type;
+      scalar_type* u_p = (scalar_type*) &u[0];
+      scalar_type* l_p = (scalar_type*) &l[0];
+      scalar_type* d_p = (scalar_type*) &d[0];
+      scalar_type* s_p = (scalar_type*) &s[0];
+
+      for(int o=0; o<LLs; o++){ // outer
+      for(int i=0; i<nsimd; i++){ //inner
+        int s   = o + i*LLs;
+        int ss  = o*nsimd + i;
+        u_p[ss] = upper[s];
+        l_p[ss] = lower[s];
+        d_p[ss] = diag[s];
+        s_p[ss] = shift_coeffs[s];
+      }}
+
+      this->M5Dcalls++;
+      this->M5Dtime -= usecond();
+
+      assert(Nc == 3);
+
+      parallel_for(int ss=0; ss<grid->oSites(); ss+=LLs){ // adds LLs
+
+        int vs     = (this->pm == 1) ? LLs-1 : 0;
+        Simd hs_00 = (this->pm == 1) ? psi[ss+vs]()(2)(0) : psi[ss+vs]()(0)(0);
+        Simd hs_01 = (this->pm == 1) ? psi[ss+vs]()(2)(1) : psi[ss+vs]()(0)(1);
+        Simd hs_02 = (this->pm == 1) ? psi[ss+vs]()(2)(2) : psi[ss+vs]()(0)(2);
+        Simd hs_10 = (this->pm == 1) ? psi[ss+vs]()(3)(0) : psi[ss+vs]()(1)(0);
+        Simd hs_11 = (this->pm == 1) ? psi[ss+vs]()(3)(1) : psi[ss+vs]()(1)(1);
+        Simd hs_12 = (this->pm == 1) ? psi[ss+vs]()(3)(2) : psi[ss+vs]()(1)(2);
+
+        for(int v=0; v<LLs; v++){
+
+          vprefetch(psi[ss+v+LLs]);
+
+          int vp = (v == LLs-1) ? 0     : v+1;
+          int vm = (v == 0)     ? LLs-1 : v-1;
+
+          Simd hp_00 = psi[ss+vp]()(2)(0);
+          Simd hp_01 = psi[ss+vp]()(2)(1);
+          Simd hp_02 = psi[ss+vp]()(2)(2);
+          Simd hp_10 = psi[ss+vp]()(3)(0);
+          Simd hp_11 = psi[ss+vp]()(3)(1);
+          Simd hp_12 = psi[ss+vp]()(3)(2);
+
+          Simd hm_00 = psi[ss+vm]()(0)(0);
+          Simd hm_01 = psi[ss+vm]()(0)(1);
+          Simd hm_02 = psi[ss+vm]()(0)(2);
+          Simd hm_10 = psi[ss+vm]()(1)(0);
+          Simd hm_11 = psi[ss+vm]()(1)(1);
+          Simd hm_12 = psi[ss+vm]()(1)(2);
+
+          if(vp <= v){
+            hp_00.v = Optimization::Rotate::tRotate<2>(hp_00.v);
+            hp_01.v = Optimization::Rotate::tRotate<2>(hp_01.v);
+            hp_02.v = Optimization::Rotate::tRotate<2>(hp_02.v);
+            hp_10.v = Optimization::Rotate::tRotate<2>(hp_10.v);
+            hp_11.v = Optimization::Rotate::tRotate<2>(hp_11.v);
+            hp_12.v = Optimization::Rotate::tRotate<2>(hp_12.v);
+          }
+
+          if(this->pm == 1 && vs <= v){
+            hs_00.v = Optimization::Rotate::tRotate<2>(hs_00.v);
+            hs_01.v = Optimization::Rotate::tRotate<2>(hs_01.v);
+            hs_02.v = Optimization::Rotate::tRotate<2>(hs_02.v);
+            hs_10.v = Optimization::Rotate::tRotate<2>(hs_10.v);
+            hs_11.v = Optimization::Rotate::tRotate<2>(hs_11.v);
+            hs_12.v = Optimization::Rotate::tRotate<2>(hs_12.v);
+          }
+
+          if(vm >= v){
+            hm_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_00.v);
+            hm_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_01.v);
+            hm_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_02.v);
+            hm_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_10.v);
+            hm_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_11.v);
+            hm_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_12.v);
+          }
+
+          if(this->pm == -1 && vs >= v){
+            hs_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_00.v);
+            hs_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_01.v);
+            hs_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_02.v);
+            hs_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_10.v);
+            hs_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_11.v);
+            hs_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_12.v);
+          }
+
+          // Can force these to real arithmetic and save 2x.
+          Simd p_00 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_00)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_00)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_00);
+          Simd p_01 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_01)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_01)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_01);
+          Simd p_02 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_02)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_02)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_02);
+          Simd p_10 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_10)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_10)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_10);
+          Simd p_11 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_11)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_11)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_11);
+          Simd p_12 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_12)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_12)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_12);
+          Simd p_20 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_00)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_00)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_00);
+          Simd p_21 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_01)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_01)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_01);
+          Simd p_22 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_02)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_02)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_02);
+          Simd p_30 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_10)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_10)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_10);
+          Simd p_31 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_11)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_11)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_11);
+          Simd p_32 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_12)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_12)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_12);
+
+          vstream(chi[ss+v]()(0)(0), p_00);
+          vstream(chi[ss+v]()(0)(1), p_01);
+          vstream(chi[ss+v]()(0)(2), p_02);
+          vstream(chi[ss+v]()(1)(0), p_10);
+          vstream(chi[ss+v]()(1)(1), p_11);
+          vstream(chi[ss+v]()(1)(2), p_12);
+          vstream(chi[ss+v]()(2)(0), p_20);
+          vstream(chi[ss+v]()(2)(1), p_21);
+          vstream(chi[ss+v]()(2)(2), p_22);
+          vstream(chi[ss+v]()(3)(0), p_30);
+          vstream(chi[ss+v]()(3)(1), p_31);
+          vstream(chi[ss+v]()(3)(2), p_32);
+        }
+      }
+
+      this->M5Dtime += usecond();
+
+    #endif
   }
 
   template<class Impl>
@@ -233,9 +398,9 @@ namespace QCD {
     int LLs = grid->_rdimensions[0];
     int nsimd = Simd::Nsimd();
 
-    Vector<iSinglet<Simd> > u(LLs);
-    Vector<iSinglet<Simd> > l(LLs);
-    Vector<iSinglet<Simd> > d(LLs);
+    Vector<iSinglet<Simd>> u(LLs);
+    Vector<iSinglet<Simd>> l(LLs);
+    Vector<iSinglet<Simd>> d(LLs);
 
     assert(Ls/LLs == nsimd);
     assert(phi.checkerboard == psi.checkerboard);
@@ -371,15 +536,179 @@ namespace QCD {
     FermionField& chi, std::vector<Coeff_t>& lower, std::vector<Coeff_t>& diag, std::vector<Coeff_t>& upper,
     std::vector<Coeff_t>& shift_coeffs)
   {
-    this->M5Ddag(psi, phi, chi, lower, diag, upper);
+    #if 0
 
-    // FIXME: possible gain from vectorizing shift operation as well?
-    Coeff_t one(1.0);
-    int Ls = this->Ls;
-    for(int s=0; s<Ls; s++){
-      if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, shift_coeffs[s], psi, Ls-1, s); }
-      else{ axpby_ssp_pminus(chi, one, chi, shift_coeffs[s], psi, 0, s); }
-    }
+      this->M5Ddag(psi, phi, chi, lower, diag, upper);
+
+      // FIXME: possible gain from vectorizing shift operation as well?
+      Coeff_t one(1.0);
+      int Ls = this->Ls;
+      for(int s=0; s<Ls; s++){
+        if(this->pm == 1){ axpby_ssp_pplus(chi, one, chi, shift_coeffs[s], psi, Ls-1, s); }
+        else{ axpby_ssp_pminus(chi, one, chi, shift_coeffs[s], psi, 0, s); }
+      }
+
+    #else
+
+      GridBase* grid = psi._grid;
+      int Ls  = this->Ls;
+      int LLs = grid->_rdimensions[0];
+      int nsimd = Simd::Nsimd();
+
+      Vector<iSinglet<Simd>> u(LLs);
+      Vector<iSinglet<Simd>> l(LLs);
+      Vector<iSinglet<Simd>> d(LLs);
+      Vector<iSinglet<Simd>> s(LLs);
+
+      assert(Ls/LLs == nsimd);
+      assert(phi.checkerboard == psi.checkerboard);
+
+      chi.checkerboard = psi.checkerboard;
+
+      // just directly address via type pun
+      typedef typename Simd::scalar_type scalar_type;
+      scalar_type* u_p = (scalar_type*) &u[0];
+      scalar_type* l_p = (scalar_type*) &l[0];
+      scalar_type* d_p = (scalar_type*) &d[0];
+      scalar_type* s_p = (scalar_type*) &s[0];
+
+      for(int o=0; o<LLs; o++){ // outer
+      for(int i=0; i<nsimd; i++){ //inner
+        int s  = o + i*LLs;
+        int ss = o*nsimd + i;
+        u_p[ss] = upper[s];
+        l_p[ss] = lower[s];
+        d_p[ss] = diag[s];
+        s_p[ss] = shift_coeffs[s];
+      }}
+
+      this->M5Dcalls++;
+      this->M5Dtime -= usecond();
+
+      parallel_for(int ss=0; ss<grid->oSites(); ss+=LLs){ // adds LLs
+
+        int vs     = (this->pm == 1) ? LLs-1 : 0;
+        Simd hs_00 = (this->pm == 1) ? psi[ss+vs]()(0)(0) : psi[ss+vs]()(2)(0);
+        Simd hs_01 = (this->pm == 1) ? psi[ss+vs]()(0)(1) : psi[ss+vs]()(2)(1);
+        Simd hs_02 = (this->pm == 1) ? psi[ss+vs]()(0)(2) : psi[ss+vs]()(2)(2);
+        Simd hs_10 = (this->pm == 1) ? psi[ss+vs]()(1)(0) : psi[ss+vs]()(3)(0);
+        Simd hs_11 = (this->pm == 1) ? psi[ss+vs]()(1)(1) : psi[ss+vs]()(3)(1);
+        Simd hs_12 = (this->pm == 1) ? psi[ss+vs]()(1)(2) : psi[ss+vs]()(3)(2);
+
+        for(int v=0; v<LLs; v++){
+
+          vprefetch(psi[ss+v+LLs]);
+
+          int vp = (v == LLs-1) ? 0     : v+1;
+          int vm = (v == 0    ) ? LLs-1 : v-1;
+
+          Simd hp_00 = psi[ss+vp]()(0)(0);
+          Simd hp_01 = psi[ss+vp]()(0)(1);
+          Simd hp_02 = psi[ss+vp]()(0)(2);
+          Simd hp_10 = psi[ss+vp]()(1)(0);
+          Simd hp_11 = psi[ss+vp]()(1)(1);
+          Simd hp_12 = psi[ss+vp]()(1)(2);
+
+          Simd hm_00 = psi[ss+vm]()(2)(0);
+          Simd hm_01 = psi[ss+vm]()(2)(1);
+          Simd hm_02 = psi[ss+vm]()(2)(2);
+          Simd hm_10 = psi[ss+vm]()(3)(0);
+          Simd hm_11 = psi[ss+vm]()(3)(1);
+          Simd hm_12 = psi[ss+vm]()(3)(2);
+
+          if (vp <= v){
+            hp_00.v = Optimization::Rotate::tRotate<2>(hp_00.v);
+            hp_01.v = Optimization::Rotate::tRotate<2>(hp_01.v);
+            hp_02.v = Optimization::Rotate::tRotate<2>(hp_02.v);
+            hp_10.v = Optimization::Rotate::tRotate<2>(hp_10.v);
+            hp_11.v = Optimization::Rotate::tRotate<2>(hp_11.v);
+            hp_12.v = Optimization::Rotate::tRotate<2>(hp_12.v);
+          }
+
+          if(this->pm == 1 && vs <= v){
+            hs_00.v = Optimization::Rotate::tRotate<2>(hs_00.v);
+            hs_01.v = Optimization::Rotate::tRotate<2>(hs_01.v);
+            hs_02.v = Optimization::Rotate::tRotate<2>(hs_02.v);
+            hs_10.v = Optimization::Rotate::tRotate<2>(hs_10.v);
+            hs_11.v = Optimization::Rotate::tRotate<2>(hs_11.v);
+            hs_12.v = Optimization::Rotate::tRotate<2>(hs_12.v);
+          }
+
+          if(vm >= v){
+            hm_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_00.v);
+            hm_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_01.v);
+            hm_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_02.v);
+            hm_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_10.v);
+            hm_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_11.v);
+            hm_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hm_12.v);
+          }
+
+          if(this->pm == -1 && vs >= v){
+            hs_00.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_00.v);
+            hs_01.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_01.v);
+            hs_02.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_02.v);
+            hs_10.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_10.v);
+            hs_11.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_11.v);
+            hs_12.v = Optimization::Rotate::tRotate<2*Simd::Nsimd()-2>(hs_12.v);
+          }
+
+          Simd p_00 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_00)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_00)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_00);
+          Simd p_01 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_01)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_01)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_01);
+          Simd p_02 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_02)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_02)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(0)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_02);
+          Simd p_10 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_10)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_10)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(0)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_10);
+          Simd p_11 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_11)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_11)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(1)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_11);
+          Simd p_12 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_12)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_12)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(1)(2)) + switcheroo<Coeff_t>::mult(u[v]()()(), hp_12);
+          Simd p_20 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_00)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_00)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_00);
+          Simd p_21 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_01)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_01)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_01);
+          Simd p_22 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_02)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(2)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_02)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_02);
+          Simd p_30 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_10)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(0)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_10)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_10);
+          Simd p_31 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_11)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(1)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_11)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_11);
+          Simd p_32 = (this->pm == 1) ? switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_12)
+                                      : switcheroo<Coeff_t>::mult(d[v]()()(), phi[ss+v]()(3)(2)) + switcheroo<Coeff_t>::mult(l[v]()()(), hm_12)
+                                                                                                 + switcheroo<Coeff_t>::mult(s[v]()()(), hs_12);
+
+          vstream(chi[ss+v]()(0)(0), p_00);
+          vstream(chi[ss+v]()(0)(1), p_01);
+          vstream(chi[ss+v]()(0)(2), p_02);
+          vstream(chi[ss+v]()(1)(0), p_10);
+          vstream(chi[ss+v]()(1)(1), p_11);
+          vstream(chi[ss+v]()(1)(2), p_12);
+          vstream(chi[ss+v]()(2)(0), p_20);
+          vstream(chi[ss+v]()(2)(1), p_21);
+          vstream(chi[ss+v]()(2)(2), p_22);
+          vstream(chi[ss+v]()(3)(0), p_30);
+          vstream(chi[ss+v]()(3)(1), p_31);
+          vstream(chi[ss+v]()(3)(2), p_32);
+
+        }
+
+      }
+
+      this->M5Dtime += usecond();
+
+    #endif
   }
 
   #ifdef AVX512

From 46f88e6d726c238b84a9b05894f7157cb6f5c2a3 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Wed, 23 Aug 2017 13:21:10 -0400
Subject: [PATCH 062/115] G-parity hand-unrolled intrinsics twist now uses one
 less permute and one less temporary

---
 lib/qcd/action/fermion/WilsonKernelsHand.cc | 66 +++++++++++++--------
 1 file changed, 42 insertions(+), 24 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index 045a2cda..6e03379e 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -78,16 +78,30 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 //else if( (distance == -1 && !perm) || (distance == 1 && perm) )
 //Pulled fermion through backwards face, GPBC on lower component
 //Need 0= 1l 0h   1= 0l 1h
-#define DO_TWIST(INTO,S,C,F, tmp1, tmp2, tmp3, tmp4)			\
+
+//1l 1h -> 1h 1l
+//0l 0h , 1h 1l -> 0l 1h 0h,1l
+#define DO_TWIST_0L_1H(INTO,S,C,F, tmp1, tmp2, tmp3) \
   permute(tmp1, ref(1)(S)(C), permute_type);				\
   exchange(tmp2,tmp3, ref(0)(S)(C), tmp1, permute_type);		\
-  permute(tmp4, tmp3, permute_type);					\
-  if( (distance == 1 && !perm) || (distance == -1 && perm) ){		\
-    INTO = F == 0 ? tmp2 : tmp4;					\
-  }else if( (distance == -1 && !perm) || (distance == 1 && perm) ){	\
-    INTO = F == 0 ? tmp4 : tmp2;					\
+  INTO = tmp2;
+
+//0l 0h -> 0h 0l
+//1l 1h, 0h 0l -> 1l 0h, 1h 0l
+#define DO_TWIST_1L_0H(INTO,S,C,F, tmp1, tmp2, tmp3)			\
+  permute(tmp1, ref(0)(S)(C), permute_type);				\
+  exchange(tmp2,tmp3, ref(1)(S)(C), tmp1, permute_type);		\
+  INTO = tmp2;
+
+#define DO_TWIST(INTO,S,C,F, tmp1, tmp2, tmp3)				\
+  if(  ( F==0 && ((distance == 1 && !perm) || (distance == -1 && perm)) ) || \
+       ( F==1 && ((distance == -1 && !perm) || (distance == 1 && perm)) ) ){ \
+    DO_TWIST_0L_1H(INTO,S,C,F,tmp1,tmp2,tmp3);				\
+  }else{								\
+    DO_TWIST_1L_0H(INTO,S,C,F,tmp1,tmp2,tmp3);				\
   }
 
+
 #define LOAD_CHI_SETUP(DIR,F)						\
   int g = F;								\
   const int direction = st._directions[DIR];				\
@@ -109,18 +123,18 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
       LOAD_CHIMU_BODY(g);						\
     }else{								\
       const int permute_type = st._grid->PermuteType(direction);	\
-      DO_TWIST(Chimu_00,0,0,F,  U_00,U_01,U_10,U_11);			\
-      DO_TWIST(Chimu_01,0,1,F,  U_20,U_21,U_00,U_01);			\
-      DO_TWIST(Chimu_02,0,2,F,  U_10,U_11,U_20,U_21);			\
-      DO_TWIST(Chimu_10,1,0,F,  U_00,U_01,U_10,U_11);			\
-      DO_TWIST(Chimu_11,1,1,F,  U_20,U_21,U_00,U_01);			\
-      DO_TWIST(Chimu_12,1,2,F,  U_10,U_11,U_20,U_21);			\
-      DO_TWIST(Chimu_20,2,0,F,  U_00,U_01,U_10,U_11);			\
-      DO_TWIST(Chimu_21,2,1,F,  U_20,U_21,U_00,U_01);			\
-      DO_TWIST(Chimu_22,2,2,F,  U_10,U_11,U_20,U_21);			\
-      DO_TWIST(Chimu_30,3,0,F,  U_00,U_01,U_10,U_11);			\
-      DO_TWIST(Chimu_31,3,1,F,  U_20,U_21,U_00,U_01);			\
-      DO_TWIST(Chimu_32,3,2,F,  U_10,U_11,U_20,U_21);			\
+      DO_TWIST(Chimu_00,0,0,F,  U_00,U_01,U_10);			\
+      DO_TWIST(Chimu_01,0,1,F,  U_11,U_20,U_21);			\
+      DO_TWIST(Chimu_02,0,2,F,  U_00,U_01,U_10);			\
+      DO_TWIST(Chimu_10,1,0,F,  U_11,U_20,U_21);			\
+      DO_TWIST(Chimu_11,1,1,F,  U_00,U_01,U_10);			\
+      DO_TWIST(Chimu_12,1,2,F,  U_11,U_20,U_21);			\
+      DO_TWIST(Chimu_20,2,0,F,  U_00,U_01,U_10);			\
+      DO_TWIST(Chimu_21,2,1,F,  U_11,U_20,U_21);			\
+      DO_TWIST(Chimu_22,2,2,F,  U_00,U_01,U_10);			\
+      DO_TWIST(Chimu_30,3,0,F,  U_11,U_20,U_21);			\
+      DO_TWIST(Chimu_31,3,1,F,  U_00,U_01,U_10);			\
+      DO_TWIST(Chimu_32,3,2,F,  U_11,U_20,U_21);			\
     } \
   }
 
@@ -132,15 +146,19 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
       LOAD_CHI_BODY(g);							\
     }else{								\
       const int permute_type = st._grid->PermuteType(direction);	\
-      DO_TWIST(Chi_00,0,0,F,  U_00,U_01,U_10,U_11);			\
-      DO_TWIST(Chi_01,0,1,F,  U_20,U_21,UChi_00,UChi_01);		\
-      DO_TWIST(Chi_02,0,2,F,  UChi_02,UChi_10,UChi_11,UChi_12);		\
-      DO_TWIST(Chi_10,1,0,F,  U_00,U_01,U_10,U_11);			\
-      DO_TWIST(Chi_11,1,1,F,  U_20,U_21,UChi_00,UChi_01);		\
-      DO_TWIST(Chi_12,1,2,F,  UChi_02,UChi_10,UChi_11,UChi_12);		\
+      DO_TWIST(Chi_00,0,0,F,  U_00,U_01,U_10);				\
+      DO_TWIST(Chi_01,0,1,F,  U_11,U_20,U_21);				\
+      DO_TWIST(Chi_02,0,2,F,  UChi_00,UChi_01,UChi_02);			\
+      DO_TWIST(Chi_10,1,0,F,  UChi_10,UChi_11,UChi_12);			\
+      DO_TWIST(Chi_11,1,1,F,  U_00,U_01,U_10);				\
+      DO_TWIST(Chi_12,1,2,F,  U_11,U_20,U_21);				\
     }									\
   }
 
+
+
+
+
 //#define LOAD_CHI_GPARITY(DIR,F) LOAD_CHI_GPARITY_IMPL(DIR,F)
 #define LOAD_CHI_GPARITY(DIR,F) LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F)
 

From a0bb8e5b4660634db2b4a7cf994b2665926b24e9 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Wed, 23 Aug 2017 14:44:40 -0400
Subject: [PATCH 063/115] Added hand-unrolled kernel implementations of all the
 other dslash precision / comms precision combinations with G-parity

---
 lib/qcd/action/fermion/WilsonKernelsHand.cc | 213 ++++++++++----------
 1 file changed, 103 insertions(+), 110 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index 6e03379e..ea04845e 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -786,120 +786,113 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D
 				    const FermionField &in,		\
 				    FermionField &out){ assert(0); }	\
 
-  HAND_SPECIALISE_EMPTY(GparityWilsonImplF);
-  //HAND_SPECIALISE_EMPTY(GparityWilsonImplD);
-  HAND_SPECIALISE_EMPTY(GparityWilsonImplFH);
-  HAND_SPECIALISE_EMPTY(GparityWilsonImplDF);
 
 
+#define HAND_SPECIALISE_GPARITY(IMPL)					\
+  template<> void							\
+  WilsonKernels<IMPL>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf, \
+				    int ss,int sU,const FermionField &in, FermionField &out) \
+  {									\
+    typedef IMPL Impl;							\
+    typedef typename Simd::scalar_type S;				\
+    typedef typename Simd::vector_type V;				\
+									\
+    HAND_DECLARATIONS(ignore);						\
+									\
+    int offset,local,perm, ptype;					\
+    StencilEntry *SE;							\
+    HAND_DOP_SITE(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+    HAND_DOP_SITE(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+  }									\
+									\
+  template<>								\
+  void WilsonKernels<IMPL>::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \
+					    int ss,int sU,const FermionField &in, FermionField &out) \
+  {									\
+    typedef IMPL Impl;							\
+    typedef typename Simd::scalar_type S;				\
+    typedef typename Simd::vector_type V;				\
+									\
+    HAND_DECLARATIONS(ignore);						\
+									\
+    StencilEntry *SE;							\
+    int offset,local,perm, ptype;					\
+    HAND_DOP_SITE_DAG(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+    HAND_DOP_SITE_DAG(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+  }									\
+									\
+  template<> void							\
+  WilsonKernels<IMPL>::HandDhopSiteInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf, \
+						     int ss,int sU,const FermionField &in, FermionField &out) \
+  {									\
+    typedef IMPL Impl;							\
+    typedef typename Simd::scalar_type S;				\
+    typedef typename Simd::vector_type V;				\
+									\
+    HAND_DECLARATIONS(ignore);						\
+									\
+    int offset,local,perm, ptype;					\
+    StencilEntry *SE;							\
+    HAND_DOP_SITE_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+    HAND_DOP_SITE_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+  }									\
+									\
+  template<>								\
+  void WilsonKernels<IMPL>::HandDhopSiteDagInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \
+							     int ss,int sU,const FermionField &in, FermionField &out) \
+  {									\
+    typedef IMPL Impl;							\
+    typedef typename Simd::scalar_type S;				\
+    typedef typename Simd::vector_type V;				\
+									\
+    HAND_DECLARATIONS(ignore);						\
+									\
+    StencilEntry *SE;							\
+    int offset,local,perm, ptype;					\
+    HAND_DOP_SITE_DAG_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+    HAND_DOP_SITE_DAG_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+  }									\
+									\
+  template<> void							\
+  WilsonKernels<IMPL>::HandDhopSiteExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf, \
+						     int ss,int sU,const FermionField &in, FermionField &out) \
+  {									\
+    typedef IMPL Impl;							\
+    typedef typename Simd::scalar_type S;				\
+    typedef typename Simd::vector_type V;				\
+									\
+    HAND_DECLARATIONS(ignore);						\
+									\
+    int offset,local,perm, ptype;					\
+    StencilEntry *SE;							\
+    int nmu=0;								\
+    HAND_DOP_SITE_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+    nmu = 0;								\
+    HAND_DOP_SITE_EXT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+  }									\
+  template<>								\
+  void WilsonKernels<IMPL>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf, \
+							     int ss,int sU,const FermionField &in, FermionField &out) \
+  {									\
+    typedef IMPL Impl;							\
+    typedef typename Simd::scalar_type S;				\
+    typedef typename Simd::vector_type V;				\
+									\
+    HAND_DECLARATIONS(ignore);						\
+									\
+    StencilEntry *SE;							\
+    int offset,local,perm, ptype;					\
+    int nmu=0;								\
+    HAND_DOP_SITE_DAG_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+    nmu = 0;								\
+    HAND_DOP_SITE_DAG_EXT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
+  }
 
 
-
-template<> void 
-WilsonKernels<GparityWilsonImplD>::HandDhopSite(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
-					  int ss,int sU,const FermionField &in, FermionField &out)
-{
-// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-  typedef GparityWilsonImplD Impl;
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  int offset,local,perm, ptype;
-  StencilEntry *SE;
-  HAND_DOP_SITE(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-  HAND_DOP_SITE(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-}
-
-template<>
-void WilsonKernels<GparityWilsonImplD>::HandDhopSiteDag(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-						  int ss,int sU,const FermionField &in, FermionField &out)
-{
-  typedef GparityWilsonImplD Impl;
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  StencilEntry *SE;
-  int offset,local,perm, ptype;
-  HAND_DOP_SITE_DAG(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-  HAND_DOP_SITE_DAG(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-}
-
-template<> void 
-WilsonKernels<GparityWilsonImplD>::HandDhopSiteInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
-					  int ss,int sU,const FermionField &in, FermionField &out)
-{
-// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-  typedef GparityWilsonImplD Impl;
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  int offset,local,perm, ptype;
-  StencilEntry *SE;
-  HAND_DOP_SITE_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-  HAND_DOP_SITE_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-}
-
-template<>
-void WilsonKernels<GparityWilsonImplD>::HandDhopSiteDagInt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-						  int ss,int sU,const FermionField &in, FermionField &out)
-{
-  typedef GparityWilsonImplD Impl;
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  StencilEntry *SE;
-  int offset,local,perm, ptype;  
-  HAND_DOP_SITE_DAG_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-  HAND_DOP_SITE_DAG_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-}
-
-template<> void 
-WilsonKernels<GparityWilsonImplD>::HandDhopSiteExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor  *buf,
-					  int ss,int sU,const FermionField &in, FermionField &out)
-{
-// T==0, Z==1, Y==2, Z==3 expect 1,2,2,2 simd layout etc...
-  typedef GparityWilsonImplD Impl;
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  int offset,local,perm, ptype;
-  StencilEntry *SE;
-  int nmu=0;
-  HAND_DOP_SITE_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-  nmu = 0;
-  HAND_DOP_SITE_EXT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);  
-}
-
-template<>
-void WilsonKernels<GparityWilsonImplD>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,DoubledGaugeField &U,SiteHalfSpinor *buf,
-						  int ss,int sU,const FermionField &in, FermionField &out)
-{
-  typedef GparityWilsonImplD Impl;
-  typedef typename Simd::scalar_type S;
-  typedef typename Simd::vector_type V;
-
-  HAND_DECLARATIONS(ignore);
-
-  StencilEntry *SE;
-  int offset,local,perm, ptype;
-  int nmu=0;
-  HAND_DOP_SITE_DAG_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-  nmu = 0;
-  HAND_DOP_SITE_DAG_EXT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY);
-}
-
-
-
+HAND_SPECIALISE_GPARITY(GparityWilsonImplF);
+HAND_SPECIALISE_GPARITY(GparityWilsonImplD);
+HAND_SPECIALISE_GPARITY(GparityWilsonImplFH);
+HAND_SPECIALISE_GPARITY(GparityWilsonImplDF);
 
 
 

From ce5df177eeccee214f3f394bbdec385f7c19caeb Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Wed, 23 Aug 2017 15:05:22 -0400
Subject: [PATCH 064/115] Removed superfluous implementation of G-parity twist
 for hand-unrolled kernel from GparityWilsonImpl

---
 lib/qcd/action/fermion/FermionOperatorImpl.h | 85 +-------------------
 lib/qcd/action/fermion/WilsonKernelsHand.cc  | 14 ----
 2 files changed, 1 insertion(+), 98 deletions(-)

diff --git a/lib/qcd/action/fermion/FermionOperatorImpl.h b/lib/qcd/action/fermion/FermionOperatorImpl.h
index 5300063b..ffb82989 100644
--- a/lib/qcd/action/fermion/FermionOperatorImpl.h
+++ b/lib/qcd/action/fermion/FermionOperatorImpl.h
@@ -478,10 +478,7 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
       
  ImplParams Params;
 
- std::vector<SiteSpinor, alignedAllocator<SiteSpinor> > tmp_full;
- std::vector<SiteHalfSpinor, alignedAllocator<SiteHalfSpinor> > tmp_half;
-
- GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p), tmp_full(GridThread::GetThreads()), tmp_half(GridThread::GetThreads()){};
+ GparityWilsonImpl(const ImplParams &p = ImplParams()) : Params(p){};
 
  bool overlapCommsCompute(void) { return Params.overlapCommsCompute; };
 
@@ -563,86 +560,6 @@ class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresent
    reg = memory;
  }
 
- template<typename SiteSpinorType>
- void GparityTwistPermute(SiteSpinorType &into,  const SiteSpinorType &from, const int direction, const int distance, const int perm, GridBase* grid){
-#if 0
-   typedef typename SiteSpinorType::scalar_object sobj;
-   sobj stmp;
-   std::vector<sobj> vals(grid->Nsimd());
-   extract(from,vals);
-   std::vector<int> icoor;
-   for(int s=0;s<grid->Nsimd();s++){
-     grid->iCoorFromIindex(icoor,s);
-     assert((icoor[direction]==0)||(icoor[direction]==1));
-
-     int permute_lane;
-     if ( distance == 1) {
-       permute_lane = icoor[direction]?1:0;
-     } else {
-       permute_lane = icoor[direction]?0:1;
-     }
-     if(perm) permute_lane = !permute_lane;                                                                                                                                                                                                
-
-     if ( permute_lane ) {
-       stmp(0) = vals[s](1);
-       stmp(1) = vals[s](0);
-       vals[s] = stmp;
-     }
-   }
-   merge(into,vals);
-#else
-   int permute_type = grid->PermuteType(direction);
-   typedef typename SiteSpinorType::vector_type vtype;
-   vtype tmp1, tmp2, tmp3, tmp4;
-
-   for(int s=0;s<Ns;s++){
-     for(int c=0;c<Nrepresentation;c++){
-       permute(tmp1, from(1)(s)(c), permute_type); //1l 1h -> 1h 1l
-       exchange(tmp2,tmp3, from(0)(s)(c), tmp1, permute_type); // 0l 0h , 1h 1l -> 0l 1h 0h,1l
-       permute(tmp4, tmp3, permute_type); //0h,1l -> 1l,0h
-              
-       if( (distance == 1 && !perm) || (distance == -1 && perm) ){
-	 //Pulled fermion through forwards face, GPBC on upper component
-	 //Need 0= 0l 1h   1= 1l 0h
-	 into(0)(s)(c) = tmp2;
-	 into(1)(s)(c) = tmp4;
-       }else if( (distance == -1 && !perm) || (distance == 1 && perm) ){
-	 //Pulled fermion through backwards face, GPBC on lower component
-	 //Need 0= 1l 0h   1= 0l 1h	 
-	 into(0)(s)(c) = tmp4;
-	 into(1)(s)(c) = tmp2;
-       }else assert(0);
-     }
-   }
-#endif
- }
-
-
- template<typename SiteSpinorType>
- const SiteSpinorType & GparityGetChi(int &g, SiteSpinorType const* in, const int dir, const int f, StencilEntry *SE, StencilImpl &st){
-   const int mmu = dir % 4;
-   const int direction = st._directions[dir];
-   const int sl = st._grid->_simd_layout[direction];
-   const int perm = SE->_permute;
-   g = f;
-   
-   if(SE->_around_the_world && Params.twists[mmu]){
-     if(sl == 1){ //not SIMD vectorized in G-parity direction so just change the flavor index accessed to implement the twist
-       g = (f+1) % 2;
-       return in[SE->_offset];
-     }else{ //SIMD vectorized in Gparity direction
-       const int me = omp_get_thread_num();
-       const int distance = st._distances[dir];
-       assert(distance == -1 || distance == 1);
-       SiteSpinorType &tmp = GparityWilsonImpl_helper::getAB<SiteSpinorType, SiteSpinor, SiteHalfSpinor>::ref(tmp_full[me], tmp_half[me]);
-       GparityTwistPermute<SiteSpinorType>(tmp, in[SE->_offset], direction, distance, perm, st._grid);
-       return tmp;
-     }
-   }else return in[SE->_offset];
- }
-
-
- 
  inline void DoubleStore(GridBase *GaugeGrid,DoubledGaugeField &Uds,const GaugeField &Umu)
  {
    conformable(Uds._grid,GaugeGrid);
diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index ea04845e..e1243304 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -59,14 +59,6 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   {const SiteHalfSpinor &ref(buf[offset]); LOAD_CHI_BODY(F); }
 
 
-//G-parity implementations using implementation method
-#define LOAD_CHIMU_GPARITY_IMPL(DIR,F)					\
-  { int g; const SiteSpinor & ref = GparityGetChi<SiteSpinor>(g,in._odata.data(),DIR,F,SE,st); LOAD_CHIMU_BODY(g); }
-
-#define LOAD_CHI_GPARITY_IMPL(DIR,F)					\
-  { int g; const SiteHalfSpinor &ref = GparityGetChi<SiteHalfSpinor>(g,buf,DIR,F,SE,st); LOAD_CHI_BODY(g); }
-
-
 //G-parity implementations using in-place intrinsic ops
 
 //1l 1h -> 1h 1l
@@ -156,13 +148,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   }
 
 
-
-
-
-//#define LOAD_CHI_GPARITY(DIR,F) LOAD_CHI_GPARITY_IMPL(DIR,F)
 #define LOAD_CHI_GPARITY(DIR,F) LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F)
-
-//#define LOAD_CHIMU_GPARITY(DIR,F) LOAD_CHIMU_GPARITY_IMPL(DIR,F)
 #define LOAD_CHIMU_GPARITY(DIR,F) LOAD_CHIMU_GPARITY_INPLACE_TWIST(DIR,F)
 
 // To splat or not to splat depends on the implementation

From edabb3577ff13df048fccb8e9fb17080531e32df Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Wed, 23 Aug 2017 16:54:06 -0400
Subject: [PATCH 065/115] Imported Benchmark_gparity

---
 benchmarks/Benchmark_gparity.cc | 190 ++++++++++++++++++++++++++++++++
 1 file changed, 190 insertions(+)
 create mode 100644 benchmarks/Benchmark_gparity.cc

diff --git a/benchmarks/Benchmark_gparity.cc b/benchmarks/Benchmark_gparity.cc
new file mode 100644
index 00000000..f6036aa8
--- /dev/null
+++ b/benchmarks/Benchmark_gparity.cc
@@ -0,0 +1,190 @@
+#include <Grid/Grid.h>
+#include <sstream>
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+template<class d>
+struct scal {
+  d internal;
+};
+
+  Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT
+  };
+
+typedef typename GparityDomainWallFermionF::FermionField GparityLatticeFermionF;
+typedef typename GparityDomainWallFermionD::FermionField GparityLatticeFermionD;
+
+
+
+int main (int argc, char ** argv)
+{
+  Grid_init(&argc,&argv);
+
+  int Ls=16;
+  for(int i=0;i<argc;i++)
+    if(std::string(argv[i]) == "-Ls"){
+      std::stringstream ss(argv[i+1]); ss >> Ls;
+    }
+
+
+  int threads = GridThread::GetThreads();
+  std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
+  std::cout<<GridLogMessage << "Ls = " << Ls << std::endl;
+
+  std::vector<int> latt4 = GridDefaultLatt();
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexF::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  std::vector<int> seeds4({1,2,3,4});
+  std::vector<int> seeds5({5,6,7,8});
+  
+  std::cout << GridLogMessage << "Initialising 4d RNG" << std::endl;
+  GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+  std::cout << GridLogMessage << "Initialising 5d RNG" << std::endl;
+  GridParallelRNG          RNG5(FGrid);  RNG5.SeedFixedIntegers(seeds5);
+  std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
+
+  GparityLatticeFermionF src   (FGrid); random(RNG5,src);
+  RealD N2 = 1.0/::sqrt(norm2(src));
+  src = src*N2;
+
+  GparityLatticeFermionF result(FGrid); result=zero;
+  GparityLatticeFermionF    ref(FGrid);    ref=zero;
+  GparityLatticeFermionF    tmp(FGrid);
+  GparityLatticeFermionF    err(FGrid);
+
+  std::cout << GridLogMessage << "Drawing gauge field" << std::endl;
+  LatticeGaugeFieldF Umu(UGrid); 
+  SU3::HotConfiguration(RNG4,Umu); 
+  std::cout << GridLogMessage << "Random gauge initialised " << std::endl;
+
+  RealD mass=0.1;
+  RealD M5  =1.8;
+
+  RealD NP = UGrid->_Nprocessors;
+  RealD NN = UGrid->NodeCount();
+
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Kernel options --dslash-generic, --dslash-unroll, --dslash-asm" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+  std::cout << GridLogMessage<< "* Benchmarking DomainWallFermion::Dhop                  "<<std::endl;
+  std::cout << GridLogMessage<< "* Vectorising space-time by "<<vComplexF::Nsimd()<<std::endl;
+#ifdef GRID_OMP
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+  if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+#endif
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+  if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+  std::cout << GridLogMessage<< "*****************************************************************" <<std::endl;
+
+
+
+  std::cout << GridLogMessage<< "* SINGLE/SINGLE"<<std::endl;
+  GparityDomainWallFermionF Dw(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  int ncall =1000;
+  if (1) {
+    FGrid->Barrier();
+    Dw.ZeroCounters();
+    Dw.Dhop(src,result,0);
+    std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      Dw.Dhop(src,result,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=2*1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    //    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
+    //    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    Dw.Report();
+  }
+
+  std::cout << GridLogMessage<< "* SINGLE/HALF"<<std::endl;
+  GparityDomainWallFermionFH DwH(Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass,M5);
+  if (1) {
+    FGrid->Barrier();
+    DwH.ZeroCounters();
+    DwH.Dhop(src,result,0);
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      DwH.Dhop(src,result,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=2*1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called half prec comms Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    DwH.Report();
+  }
+
+  GridCartesian         * UGrid_d   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexD::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid_d = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid_d);
+  GridCartesian         * FGrid_d   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid_d);
+  GridRedBlackCartesian * FrbGrid_d = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid_d);
+
+  
+  std::cout << GridLogMessage<< "* DOUBLE/DOUBLE"<<std::endl;
+  GparityLatticeFermionD src_d(FGrid_d);
+  precisionChange(src_d,src);
+
+  LatticeGaugeFieldD Umu_d(UGrid_d); 
+  precisionChange(Umu_d,Umu);
+
+  GparityLatticeFermionD result_d(FGrid_d);
+
+  GparityDomainWallFermionD DwD(Umu_d,*FGrid_d,*FrbGrid_d,*UGrid_d,*UrbGrid_d,mass,M5);
+  if (1) {
+    FGrid_d->Barrier();
+    DwD.ZeroCounters();
+    DwD.Dhop(src_d,result_d,0);
+    std::cout<<GridLogMessage<<"Called warmup"<<std::endl;
+    double t0=usecond();
+    for(int i=0;i<ncall;i++){
+      __SSC_START;
+      DwD.Dhop(src_d,result_d,0);
+      __SSC_STOP;
+    }
+    double t1=usecond();
+    FGrid_d->Barrier();
+    
+    double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+    double flops=2*1344*volume*ncall;
+
+    std::cout<<GridLogMessage << "Called Dw "<<ncall<<" times in "<<t1-t0<<" us"<<std::endl;
+    //    std::cout<<GridLogMessage << "norm result "<< norm2(result)<<std::endl;
+    //    std::cout<<GridLogMessage << "norm ref    "<< norm2(ref)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s =   "<< flops/(t1-t0)<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per rank =  "<< flops/(t1-t0)/NP<<std::endl;
+    std::cout<<GridLogMessage << "mflop/s per node =  "<< flops/(t1-t0)/NN<<std::endl;
+    DwD.Report();
+  }
+
+  Grid_finalize();
+}
+

From 5fa386ddc96fc13146e528ea4d3d92ec9552c49e Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 24 Aug 2017 10:17:52 +0100
Subject: [PATCH 066/115] FFT test compile fixed

---
 lib/qcd/utils/GaugeFix.h    | 3 +++
 tests/core/Test_fft_gfix.cc | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/lib/qcd/utils/GaugeFix.h b/lib/qcd/utils/GaugeFix.h
index f2ea1aa2..c4ea31aa 100644
--- a/lib/qcd/utils/GaugeFix.h
+++ b/lib/qcd/utils/GaugeFix.h
@@ -26,6 +26,8 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 //#include <Grid/Grid.h>
 
+#ifndef GRID_QCD_GAUGE_FIX_H
+#define GRID_QCD_GAUGE_FIX_H
 namespace Grid {
 namespace QCD {
 
@@ -188,3 +190,4 @@ class FourierAcceleratedGaugeFixer  : public Gimpl {
 
 }
 }
+#endif
diff --git a/tests/core/Test_fft_gfix.cc b/tests/core/Test_fft_gfix.cc
index 9732eb85..916c4b0b 100644
--- a/tests/core/Test_fft_gfix.cc
+++ b/tests/core/Test_fft_gfix.cc
@@ -28,6 +28,9 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
     /*  END LEGAL */
 #include <Grid/Grid.h>
 
+using namespace Grid;
+using namespace Grid::QCD;
+
 int main (int argc, char ** argv)
 {
   std::vector<int> seeds({1,2,3,4});
@@ -82,6 +85,7 @@ int main (int argc, char ** argv)
 
   Uorg = Uorg - Umu;
   std::cout << " Norm Difference "<< norm2(Uorg) << std::endl;
+  std::cout << " Norm "<< norm2(Umu) << std::endl;
 
 
   std::cout<< "*****************************************************************" <<std::endl;

From 102ea9ae668a1c8eef506113348355e9d78fd522 Mon Sep 17 00:00:00 2001
From: Antonin Portelli <antonin.portelli@me.com>
Date: Thu, 24 Aug 2017 18:17:09 +0100
Subject: [PATCH 067/115] CI update

---
 .travis.yml | 68 -----------------------------------------------------
 README.md   | 16 +------------
 2 files changed, 1 insertion(+), 83 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 64dae823..7d8203ce 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,68 +9,6 @@ matrix:
     - os:        osx
       osx_image: xcode8.3
       compiler: clang
-    - compiler: gcc
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-4.9
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: VERSION=-4.9
-    - compiler: gcc
-      dist: trusty
-      sudo: required
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-5
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: VERSION=-5
-    - compiler: clang
-      dist: trusty
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-4.8
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: CLANG_LINK=http://llvm.org/releases/3.8.0/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-    - compiler: clang
-      dist: trusty
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-4.8
-            - libmpfr-dev
-            - libgmp-dev
-            - libmpc-dev
-            - libopenmpi-dev
-            - openmpi-bin
-            - binutils-dev
-      env: CLANG_LINK=http://llvm.org/releases/3.7.0/clang+llvm-3.7.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
       
 before_install:
     - export GRIDDIR=`pwd`
@@ -106,9 +44,3 @@ script:
     - make -j4
     - ./benchmarks/Benchmark_dwf --threads 1 --debug-signals
     - make check
-    - echo make clean
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then ../configure --enable-precision=single --enable-simd=SSE4 --enable-comms=mpi-auto ; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then make -j4; fi
-    - if [[ "$TRAVIS_OS_NAME" == "linux" ]] && [[ "$CC" == "clang" ]]; then mpirun.openmpi -n 2 ./benchmarks/Benchmark_dwf --threads 1 --mpi 2.1.1.1; fi
-
-
diff --git a/README.md b/README.md
index 1e0988f3..13dd6996 100644
--- a/README.md
+++ b/README.md
@@ -1,18 +1,4 @@
-# Grid
-<table>
-<tr>
-    <td>Last stable release</td>
-    <td><a href="https://travis-ci.org/paboyle/Grid">
-    <img src="https://travis-ci.org/paboyle/Grid.svg?branch=master"></a>
-    </td>
-</tr>
-<tr>
-    <td>Development branch</td>
-    <td><a href="https://travis-ci.org/paboyle/Grid">
-    <img src="https://travis-ci.org/paboyle/Grid.svg?branch=develop"></a>
-    </td>
-</tr>
-</table>
+# Grid [![Teamcity status](http://ci.cliath.ph.ed.ac.uk/app/rest/builds/aggregated/strob:(buildType:(affectedProject(id:Grid)),branch:name:develop)/statusIcon.svg)](http://ci.cliath.ph.ed.ac.uk/project.html?projectId=Grid&tab=projectOverview) [![Travis status](https://travis-ci.org/paboyle/Grid.svg?branch=develop)](https://travis-ci.org/paboyle/Grid)
 
 **Data parallel C++ mathematical object library.**
 

From 34a9aeb331b06e4a3fc8562afbe518039af55008 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Thu, 24 Aug 2017 13:53:50 -0700
Subject: [PATCH 068/115] Reduced number of if-statement evaluations in
 G-parity unrolled kernel

---
 lib/qcd/action/fermion/WilsonKernelsHand.cc | 62 +++++++++++++++------
 1 file changed, 44 insertions(+), 18 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index e1243304..ca93916b 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -115,18 +115,34 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
       LOAD_CHIMU_BODY(g);						\
     }else{								\
       const int permute_type = st._grid->PermuteType(direction);	\
-      DO_TWIST(Chimu_00,0,0,F,  U_00,U_01,U_10);			\
-      DO_TWIST(Chimu_01,0,1,F,  U_11,U_20,U_21);			\
-      DO_TWIST(Chimu_02,0,2,F,  U_00,U_01,U_10);			\
-      DO_TWIST(Chimu_10,1,0,F,  U_11,U_20,U_21);			\
-      DO_TWIST(Chimu_11,1,1,F,  U_00,U_01,U_10);			\
-      DO_TWIST(Chimu_12,1,2,F,  U_11,U_20,U_21);			\
-      DO_TWIST(Chimu_20,2,0,F,  U_00,U_01,U_10);			\
-      DO_TWIST(Chimu_21,2,1,F,  U_11,U_20,U_21);			\
-      DO_TWIST(Chimu_22,2,2,F,  U_00,U_01,U_10);			\
-      DO_TWIST(Chimu_30,3,0,F,  U_11,U_20,U_21);			\
-      DO_TWIST(Chimu_31,3,1,F,  U_00,U_01,U_10);			\
-      DO_TWIST(Chimu_32,3,2,F,  U_11,U_20,U_21);			\
+      if(  ( F==0 && ((distance == 1 && !perm) || (distance == -1 && perm)) ) || \
+	   ( F==1 && ((distance == -1 && !perm) || (distance == 1 && perm)) ) ){ \
+	DO_TWIST_0L_1H(Chimu_00,0,0,F,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_01,0,1,F,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_02,0,2,F,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_10,1,0,F,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_11,1,1,F,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_12,1,2,F,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_20,2,0,F,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_21,2,1,F,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_22,2,2,F,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_30,3,0,F,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_31,3,1,F,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_32,3,2,F,  U_11,U_20,U_21);		\
+      }else{								\
+	DO_TWIST_1L_0H(Chimu_00,0,0,F,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_01,0,1,F,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_02,0,2,F,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_10,1,0,F,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_11,1,1,F,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_12,1,2,F,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_20,2,0,F,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_21,2,1,F,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_22,2,2,F,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_30,3,0,F,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_31,3,1,F,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_32,3,2,F,  U_11,U_20,U_21);		\
+      } \
     } \
   }
 
@@ -138,12 +154,22 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
       LOAD_CHI_BODY(g);							\
     }else{								\
       const int permute_type = st._grid->PermuteType(direction);	\
-      DO_TWIST(Chi_00,0,0,F,  U_00,U_01,U_10);				\
-      DO_TWIST(Chi_01,0,1,F,  U_11,U_20,U_21);				\
-      DO_TWIST(Chi_02,0,2,F,  UChi_00,UChi_01,UChi_02);			\
-      DO_TWIST(Chi_10,1,0,F,  UChi_10,UChi_11,UChi_12);			\
-      DO_TWIST(Chi_11,1,1,F,  U_00,U_01,U_10);				\
-      DO_TWIST(Chi_12,1,2,F,  U_11,U_20,U_21);				\
+      if(  ( F==0 && ((distance == 1 && !perm) || (distance == -1 && perm)) ) || \
+	   ( F==1 && ((distance == -1 && !perm) || (distance == 1 && perm)) ) ){ \
+	DO_TWIST_0L_1H(Chi_00,0,0,F,  U_00,U_01,U_10);			\
+	DO_TWIST_0L_1H(Chi_01,0,1,F,  U_11,U_20,U_21);			\
+	DO_TWIST_0L_1H(Chi_02,0,2,F,  UChi_00,UChi_01,UChi_02);		\
+	DO_TWIST_0L_1H(Chi_10,1,0,F,  UChi_10,UChi_11,UChi_12);		\
+	DO_TWIST_0L_1H(Chi_11,1,1,F,  U_00,U_01,U_10);			\
+	DO_TWIST_0L_1H(Chi_12,1,2,F,  U_11,U_20,U_21);			\
+      }else{								\
+	DO_TWIST_1L_0H(Chi_00,0,0,F,  U_00,U_01,U_10);			\
+	DO_TWIST_1L_0H(Chi_01,0,1,F,  U_11,U_20,U_21);			\
+	DO_TWIST_1L_0H(Chi_02,0,2,F,  UChi_00,UChi_01,UChi_02);		\
+	DO_TWIST_1L_0H(Chi_10,1,0,F,  UChi_10,UChi_11,UChi_12);		\
+	DO_TWIST_1L_0H(Chi_11,1,1,F,  U_00,U_01,U_10);			\
+	DO_TWIST_1L_0H(Chi_12,1,2,F,  U_11,U_20,U_21);			\
+      }									\
     }									\
   }
 

From c3b1263e75212356fc1aa061cd226db70f4f00fc Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Fri, 25 Aug 2017 09:25:54 +0100
Subject: [PATCH 069/115] Benchmark prep

---
 benchmarks/Benchmark_ITT.cc               | 322 +++++++++++++++++++---
 benchmarks/Benchmark_comms.cc             |  30 +-
 lib/allocator/AlignedAllocator.h          |   5 +
 lib/communicator/Communicator_base.cc     |   6 +-
 lib/communicator/Communicator_mpi3.cc     |   5 +-
 lib/communicator/Communicator_mpit.cc     |  19 +-
 lib/qcd/action/fermion/CayleyFermion5D.cc |  12 +-
 lib/qcd/action/fermion/WilsonCompressor.h |  41 ++-
 lib/qcd/action/fermion/WilsonFermion5D.cc |  11 +
 lib/stencil/Stencil.h                     | 114 +++++++-
 10 files changed, 494 insertions(+), 71 deletions(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 9bf7d0a5..c5226ee1 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -32,6 +32,19 @@ using namespace std;
 using namespace Grid;
 using namespace Grid::QCD;
 
+typedef WilsonFermion5D<DomainWallVec5dImplR> WilsonFermion5DR;
+typedef WilsonFermion5D<DomainWallVec5dImplF> WilsonFermion5DF;
+typedef WilsonFermion5D<DomainWallVec5dImplD> WilsonFermion5DD;
+
+
+std::vector<int> L_list;
+std::vector<int> Ls_list;
+std::vector<double> mflop_list;
+
+double mflop_ref;
+double mflop_ref_err;
+
+int NN_global;
 
 struct time_statistics{
   double mean;
@@ -95,13 +108,15 @@ public:
 
   static void Comms(void)
   {
-    int Nloop=100;
+    int Nloop=1000;
     int nmu=0;
     int maxlat=32;
 
     std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplexD::Nsimd());
     std::vector<int> mpi_layout  = GridDefaultMpi();
 
+    for(int mu=0;mu<Nd;mu++) if (mpi_layout[mu]>1) nmu++;
+
     std::vector<double> t_time(Nloop);
     time_statistics timestat;
 
@@ -133,13 +148,14 @@ public:
 	  bzero((void *)rbuf[d],lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD));
 	}
 
-	int ncomm;
 	int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+	int ncomm;
 	double dbytes;
+	std::vector<double> times(Nloop);
 	for(int i=0;i<Nloop;i++){
+
 	  double start=usecond();
 
-	  std::vector<CartesianCommunicator::CommsRequest_t> requests;
 	  dbytes=0;
 	  ncomm=0;
 
@@ -150,7 +166,6 @@ public:
 
 	    if (mpi_layout[mu]>1 ) {
 	        
-	      ncomm++;
 	      int xmit_to_rank;
 	      int recv_from_rank;
 	      if ( dir == mu ) { 
@@ -160,18 +175,18 @@ public:
 		int comm_proc = mpi_layout[mu]-1;
 		Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	      }
-#if 1
-	      tbytes= Grid.StencilSendToRecvFromBegin(requests,
-						      (void *)&xbuf[dir][0],
-						      xmit_to_rank,
-						      (void *)&rbuf[dir][0],
-						      recv_from_rank,
-						      bytes,dir);
-	      Grid.StencilSendToRecvFromComplete(requests,dir);
-#endif
-	      requests.resize(0);
-
+	      tbytes= Grid.StencilSendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
+						 (void *)&rbuf[dir][0], recv_from_rank,
+						 bytes,dir);
+	  
+#ifdef GRID_OMP
 #pragma omp atomic
+#endif
+	      ncomm++;
+
+#ifdef GRID_OMP
+#pragma omp atomic
+#endif
 	      dbytes+=tbytes;
 	    }
 	  }
@@ -181,13 +196,15 @@ public:
 	}
 
 	timestat.statistics(t_time);
+	//	for(int i=0;i<t_time.size();i++){
+	  //	  std::cout << i<<" "<<t_time[i]<<std::endl;
+	//	}
 
 	dbytes=dbytes*ppn;
 	double xbytes    = dbytes*0.5;
 	double rbytes    = dbytes*0.5;
 	double bidibytes = dbytes;
 
-
 	std::cout<<GridLogMessage << std::setw(4) << lat<<"\t"<<Ls<<"\t"
 		 <<std::setw(11) << bytes<< std::fixed << std::setprecision(1) << std::setw(7)
 		 <<std::right<< xbytes/timestat.mean<<"  "<< xbytes*timestat.err/(timestat.mean*timestat.mean)<< " "
@@ -196,7 +213,8 @@ public:
 		 << bidibytes/timestat.max << " " << bidibytes/timestat.min << std::endl;
 
  
-      }
+	
+	    }
     }    
 
     return;
@@ -218,7 +236,7 @@ public:
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
   
   uint64_t lmax=48;
-#define NLOOP (50*lmax*lmax*lmax*lmax/lat/lat/lat/lat)
+#define NLOOP (100*lmax*lmax*lmax*lmax/lat/lat/lat/lat)
 
     GridSerialRNG          sRNG;      sRNG.SeedFixedIntegers(std::vector<int>({45,12,81,9}));
     for(int lat=8;lat<=lmax;lat+=4){
@@ -253,8 +271,7 @@ public:
     }
   };
 
-
-  static void DWF(int Ls,int L)
+  static double DWF5(int Ls,int L)
   {
     RealD mass=0.1;
     RealD M5  =1.8;
@@ -262,6 +279,7 @@ public:
     double mflops;
     double mflops_best = 0;
     double mflops_worst= 0;
+    std::vector<double> mflops_all;
 
     ///////////////////////////////////////////////////////
     // Set/Get the layout & grid size
@@ -274,6 +292,189 @@ public:
 								       GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
     uint64_t NP = TmpGrid->RankCount();
     uint64_t NN = TmpGrid->NodeCount();
+    NN_global=NN;
+    uint64_t SHM=NP/NN;
+
+    std::vector<int> internal;
+    if      ( SHM == 1 )   internal = std::vector<int>({1,1,1,1});
+    else if ( SHM == 2 )   internal = std::vector<int>({2,1,1,1});
+    else if ( SHM == 4 )   internal = std::vector<int>({2,2,1,1});
+    else if ( SHM == 8 )   internal = std::vector<int>({2,2,2,1});
+    else assert(0);
+
+    std::vector<int> nodes({mpi[0]/internal[0],mpi[1]/internal[1],mpi[2]/internal[2],mpi[3]/internal[3]});
+    std::vector<int> latt4({local[0]*nodes[0],local[1]*nodes[1],local[2]*nodes[2],local[3]*nodes[3]});
+
+    ///////// Welcome message ////////////
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << "Benchmark DWF Ls vec on "<<L<<"^4 local volume "<<std::endl;
+    std::cout<<GridLogMessage << "* Global volume  : "<<GridCmdVectorIntToString(latt4)<<std::endl;
+    std::cout<<GridLogMessage << "* Ls             : "<<Ls<<std::endl;
+    std::cout<<GridLogMessage << "* MPI ranks      : "<<GridCmdVectorIntToString(mpi)<<std::endl;
+    std::cout<<GridLogMessage << "* Intranode      : "<<GridCmdVectorIntToString(internal)<<std::endl;
+    std::cout<<GridLogMessage << "* nodes          : "<<GridCmdVectorIntToString(nodes)<<std::endl;
+    std::cout<<GridLogMessage << "* Using "<<threads<<" threads"<<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+    ///////// Lattice Init ////////////
+    GridCartesian         * UGrid    = SpaceTimeGrid::makeFourDimGrid(latt4, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    GridRedBlackCartesian * UrbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+    GridCartesian         * sUGrid   = SpaceTimeGrid::makeFourDimDWFGrid(latt4,GridDefaultMpi());
+    GridRedBlackCartesian * sUrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(sUGrid);
+    GridCartesian         * sFGrid   = SpaceTimeGrid::makeFiveDimDWFGrid(Ls,UGrid);
+    GridRedBlackCartesian * sFrbGrid = SpaceTimeGrid::makeFiveDimDWFRedBlackGrid(Ls,UGrid);
+
+    ///////// RNG Init ////////////
+    std::vector<int> seeds4({1,2,3,4});
+    std::vector<int> seeds5({5,6,7,8});
+    GridParallelRNG          RNG4(UGrid);  RNG4.SeedFixedIntegers(seeds4);
+    GridParallelRNG          RNG5(sFGrid);  RNG5.SeedFixedIntegers(seeds5);
+    std::cout << GridLogMessage << "Initialised RNGs" << std::endl;
+
+    ///////// Source preparation ////////////
+    LatticeFermion src   (sFGrid); random(RNG5,src);
+    LatticeFermion tmp   (sFGrid);
+
+    RealD N2 = 1.0/::sqrt(norm2(src));
+    src = src*N2;
+    
+    LatticeGaugeField Umu(UGrid);  SU3::HotConfiguration(RNG4,Umu); 
+
+    WilsonFermion5DR sDw(Umu,*sFGrid,*sFrbGrid,*sUGrid,*sUrbGrid,M5);
+    LatticeFermion src_e (sFrbGrid);
+    LatticeFermion src_o (sFrbGrid);
+    LatticeFermion r_e   (sFrbGrid);
+    LatticeFermion r_o   (sFrbGrid);
+    LatticeFermion r_eo  (sFGrid);
+    LatticeFermion err   (sFGrid);
+    {
+
+      pickCheckerboard(Even,src_e,src);
+      pickCheckerboard(Odd,src_o,src);
+
+#if defined(AVX512) 
+      const int num_cases = 6;
+      std::string fmt("A/S ; A/O ; U/S ; U/O ; G/S ; G/O ");
+#else
+      const int num_cases = 4;
+      std::string fmt("U/S ; U/O ; G/S ; G/O ");
+#endif
+      controls Cases [] = {
+#ifdef AVX512
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+#endif
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptGeneric   , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  }
+      }; 
+
+      for(int c=0;c<num_cases;c++) {
+
+	QCD::WilsonKernelsStatic::Comms = Cases[c].CommsOverlap;
+	QCD::WilsonKernelsStatic::Opt   = Cases[c].Opt;
+	CartesianCommunicator::SetCommunicatorPolicy(Cases[c].CommsAsynch);
+
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptGeneric   ) std::cout << GridLogMessage<< "* Using GENERIC Nc WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptHandUnroll) std::cout << GridLogMessage<< "* Using Nc=3       WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Opt == WilsonKernelsStatic::OptInlineAsm ) std::cout << GridLogMessage<< "* Using Asm Nc=3   WilsonKernels" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsAndCompute ) std::cout << GridLogMessage<< "* Using Overlapped Comms/Compute" <<std::endl;
+	if ( WilsonKernelsStatic::Comms == WilsonKernelsStatic::CommsThenCompute) std::cout << GridLogMessage<< "* Using sequential comms compute" <<std::endl;
+	if ( sizeof(Real)==4 )   std::cout << GridLogMessage<< "* SINGLE precision "<<std::endl;
+	if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
+	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+	int nwarm = 200;
+	double t0=usecond();
+	sFGrid->Barrier();
+	for(int i=0;i<nwarm;i++){
+	  sDw.DhopEO(src_o,r_e,DaggerNo);
+	}
+	sFGrid->Barrier();
+	double t1=usecond();
+	//	uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0);
+	//	if (ncall < 500) ncall = 500;
+	uint64_t ncall = 1000;
+
+	sFGrid->Broadcast(0,&ncall,sizeof(ncall));
+
+	//	std::cout << GridLogMessage << " Estimate " << ncall << " calls per second"<<std::endl;
+	sDw.ZeroCounters();
+
+	time_statistics timestat;
+	std::vector<double> t_time(ncall);
+	for(uint64_t i=0;i<ncall;i++){
+	  t0=usecond();
+	  sDw.DhopEO(src_o,r_e,DaggerNo);
+	  t1=usecond();
+	  t_time[i] = t1-t0;
+	}
+	sFGrid->Barrier();
+	
+	double volume=Ls;  for(int mu=0;mu<Nd;mu++) volume=volume*latt4[mu];
+	double flops=(1344.0*volume)/2;
+	double mf_hi, mf_lo, mf_err;
+
+	timestat.statistics(t_time);
+	mf_hi = flops/timestat.min;
+	mf_lo = flops/timestat.max;
+	mf_err= flops/timestat.min * timestat.err/timestat.mean;
+
+	mflops = flops/timestat.mean;
+	mflops_all.push_back(mflops);
+	if ( mflops_best == 0   ) mflops_best = mflops;
+	if ( mflops_worst== 0   ) mflops_worst= mflops;
+	if ( mflops>mflops_best ) mflops_best = mflops;
+	if ( mflops<mflops_worst) mflops_worst= mflops;
+
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"sDeo mflop/s =   "<< mflops << " ("<<mf_err<<") " << mf_lo<<"-"<<mf_hi <<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"sDeo mflop/s per rank   "<< mflops/NP<<std::endl;
+	std::cout<<GridLogMessage << std::fixed << std::setprecision(1)<<"sDeo mflop/s per node   "<< mflops/NN<<std::endl;
+
+	sDw.Report();
+
+      }
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " sDeo Best  mflop/s        =   "<< mflops_best << " ; " << mflops_best/NN<<" per node " <<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " sDeo Worst mflop/s        =   "<< mflops_worst<< " ; " << mflops_worst/NN<<" per node " <<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< mflops_worst/mflops_best <<std::endl;
+      std::cout<<GridLogMessage <<fmt << std::endl;
+      std::cout<<GridLogMessage ;
+
+      for(int i=0;i<mflops_all.size();i++){
+	std::cout<<mflops_all[i]/NN<<" ; " ;
+      }
+      std::cout<<std::endl;
+      std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+    }
+    return mflops_best;
+  }
+
+  static double DWF(int Ls,int L)
+  {
+    RealD mass=0.1;
+    RealD M5  =1.8;
+
+    double mflops;
+    double mflops_best = 0;
+    double mflops_worst= 0;
+    std::vector<double> mflops_all;
+
+    ///////////////////////////////////////////////////////
+    // Set/Get the layout & grid size
+    ///////////////////////////////////////////////////////
+    int threads = GridThread::GetThreads();
+    std::vector<int> mpi = GridDefaultMpi(); assert(mpi.size()==4);
+    std::vector<int> local({L,L,L,L});
+
+    GridCartesian         * TmpGrid   = SpaceTimeGrid::makeFourDimGrid(std::vector<int>({64,64,64,64}), 
+								       GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+    uint64_t NP = TmpGrid->RankCount();
+    uint64_t NN = TmpGrid->NodeCount();
+    NN_global=NN;
     uint64_t SHM=NP/NN;
 
     std::vector<int> internal;
@@ -364,13 +565,15 @@ public:
 
 #if defined(AVX512) 
       const int num_cases = 6;
+      std::string fmt("A/S ; A/O ; U/S ; U/O ; G/S ; G/O ");
 #else
       const int num_cases = 4;
+      std::string fmt("U/S ; U/O ; G/S ; G/O ");
 #endif
       controls Cases [] = {
 #ifdef AVX512
-	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
 	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
+	{ QCD::WilsonKernelsStatic::OptInlineAsm , QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
 #endif
 	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsThenCompute ,CartesianCommunicator::CommunicatorPolicySequential  },
 	{ QCD::WilsonKernelsStatic::OptHandUnroll, QCD::WilsonKernelsStatic::CommsAndCompute  ,CartesianCommunicator::CommunicatorPolicySequential  },
@@ -394,7 +597,7 @@ public:
 	if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
 	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
-	int nwarm = 10;
+	int nwarm = 200;
 	double t0=usecond();
 	FGrid->Barrier();
 	for(int i=0;i<nwarm;i++){
@@ -402,7 +605,10 @@ public:
 	}
 	FGrid->Barrier();
 	double t1=usecond();
-	uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0);
+	//	uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0);
+	//	if (ncall < 500) ncall = 500;
+	uint64_t ncall = 1000;
+
 	FGrid->Broadcast(0,&ncall,sizeof(ncall));
 
 	//	std::cout << GridLogMessage << " Estimate " << ncall << " calls per second"<<std::endl;
@@ -428,7 +634,7 @@ public:
 	mf_err= flops/timestat.min * timestat.err/timestat.mean;
 
 	mflops = flops/timestat.mean;
-
+	mflops_all.push_back(mflops);
 	if ( mflops_best == 0   ) mflops_best = mflops;
 	if ( mflops_worst== 0   ) mflops_worst= mflops;
 	if ( mflops>mflops_best ) mflops_best = mflops;
@@ -450,12 +656,20 @@ public:
 
       }
       std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Best  mflop/s        =   "<< mflops_best <<std::endl;
-      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Worst mflop/s        =   "<< mflops_worst<<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Best  mflop/s        =   "<< mflops_best << " ; " << mflops_best/NN<<" per node " <<std::endl;
+      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Worst mflop/s        =   "<< mflops_worst<< " ; " << mflops_worst/NN<<" per node " <<std::endl;
       std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< mflops_worst/mflops_best <<std::endl;
+      std::cout<<GridLogMessage <<fmt << std::endl;
+      std::cout<<GridLogMessage ;
+
+      for(int i=0;i<mflops_all.size();i++){
+	std::cout<<mflops_all[i]/NN<<" ; " ;
+      }
+      std::cout<<std::endl;
       std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
     }
+    return mflops_best;
   }
 
 };
@@ -493,26 +707,66 @@ int main (int argc, char ** argv)
     // empty for now
   }
 
+  int sel=2;
+  std::vector<int> L_list({8,12,16,24});
+  std::vector<double> wilson;
+  std::vector<double> dwf4;
+  std::vector<double> dwf5;
+
   if ( do_wilson ) {
     int Ls=1;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
     std::cout<<GridLogMessage << " Wilson dslash 4D vectorised" <<std::endl;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-    Benchmark::DWF(Ls,16);
-    Benchmark::DWF(Ls,24);
-    Benchmark::DWF(Ls,32);
+    for(int l=0;l<L_list.size();l++){
+      wilson.push_back(Benchmark::DWF(1,L_list[l]));
+    }
   }
 
+  int Ls=16;
   if ( do_dwf ) {
-    int Ls=16;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
     std::cout<<GridLogMessage << " Domain wall dslash 4D vectorised" <<std::endl;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-    Benchmark::DWF(Ls,8);
-    Benchmark::DWF(Ls,12);
-    Benchmark::DWF(Ls,16);
-    Benchmark::DWF(Ls,24);
+    for(int l=0;l<L_list.size();l++){
+      dwf4.push_back(Benchmark::DWF(Ls,L_list[l]));
+    }
   }
 
+  if ( do_dwf ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Domain wall dslash 4D vectorised" <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    for(int l=0;l<L_list.size();l++){
+      dwf5.push_back(Benchmark::DWF5(Ls,L_list[l]));
+    }
+
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " Summary table Ls="<<Ls <<std::endl;
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << "L \t\t Wilson \t DWF4 \t DWF5 " <<std::endl;
+  for(int l=0;l<L_list.size();l++){
+    std::cout<<GridLogMessage << L_list[l] <<" \t\t "<< wilson[l]<<" \t "<<dwf4[l]<<" \t "<<dwf5[l] <<std::endl;
+  }
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " Per Node Summary table Ls="<<Ls <<std::endl;
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  int NN=NN_global;
+  std::cout<<GridLogMessage << " L \t\t Wilson\t\t DWF4  \t\t DWF5 " <<std::endl;
+  for(int l=0;l<L_list.size();l++){
+    std::cout<<GridLogMessage << L_list[l] <<" \t\t "<< wilson[l]/NN<<" \t "<<dwf4[l]/NN<<" \t "<<dwf5[l] /NN<<std::endl;
+  }
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  std::cout<<GridLogMessage << " Comparison point result: "  << dwf4[sel]/NN <<std::endl;
+  std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+
+
+  }
+
+
   Grid_finalize();
 }
diff --git a/benchmarks/Benchmark_comms.cc b/benchmarks/Benchmark_comms.cc
index 491fba1e..a270e3fa 100644
--- a/benchmarks/Benchmark_comms.cc
+++ b/benchmarks/Benchmark_comms.cc
@@ -92,11 +92,16 @@ int main (int argc, char ** argv)
       RealD Nnode = Grid.NodeCount();
       RealD ppn = Nrank/Nnode;
 
-      Vector<Vector<HalfSpinColourVectorD> > xbuf(8,Vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
-      Vector<Vector<HalfSpinColourVectorD> > rbuf(8,Vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      std::vector<Vector<HalfSpinColourVectorD> > xbuf(8);	
+      std::vector<Vector<HalfSpinColourVectorD> > rbuf(8);
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
+      for(int mu=0;mu<8;mu++){
+	xbuf[mu].resize(lat*lat*lat*Ls);
+	rbuf[mu].resize(lat*lat*lat*Ls);
+	//	std::cout << " buffers " << std::hex << (uint64_t)&xbuf[mu][0] <<" " << (uint64_t)&rbuf[mu][0] <<std::endl;
+      }
 
       for(int i=0;i<Nloop;i++){
       double start=usecond();
@@ -112,7 +117,6 @@ int main (int argc, char ** argv)
 	    int comm_proc=1;
 	    int xmit_to_rank;
 	    int recv_from_rank;
-	    
 	    Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	    Grid.SendToRecvFromBegin(requests,
 				   (void *)&xbuf[mu][0],
@@ -172,9 +176,14 @@ int main (int argc, char ** argv)
       RealD Nnode = Grid.NodeCount();
       RealD ppn = Nrank/Nnode;
 
-      Vector<Vector<HalfSpinColourVectorD> > xbuf(8,Vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
-      Vector<Vector<HalfSpinColourVectorD> > rbuf(8,Vector<HalfSpinColourVectorD>(lat*lat*lat*Ls));
+      std::vector<Vector<HalfSpinColourVectorD> > xbuf(8);
+      std::vector<Vector<HalfSpinColourVectorD> > rbuf(8);
 
+      for(int mu=0;mu<8;mu++){
+	xbuf[mu].resize(lat*lat*lat*Ls);
+	rbuf[mu].resize(lat*lat*lat*Ls);
+	//	std::cout << " buffers " << std::hex << (uint64_t)&xbuf[mu][0] <<" " << (uint64_t)&rbuf[mu][0] <<std::endl;
+      }
 
       int ncomm;
       int bytes=lat*lat*lat*Ls*sizeof(HalfSpinColourVectorD);
@@ -493,14 +502,9 @@ int main (int argc, char ** argv)
 	      int comm_proc = mpi_layout[mu]-1;
 	      Grid.ShiftedRanks(mu,comm_proc,xmit_to_rank,recv_from_rank);
 	    }
-	    tbytes= Grid.StencilSendToRecvFromBegin(requests,
-						    (void *)&xbuf[dir][0],
-						    xmit_to_rank,
-						    (void *)&rbuf[dir][0],
-						    recv_from_rank,
-						    bytes,dir);
-	    Grid.StencilSendToRecvFromComplete(requests,dir);
-	    requests.resize(0);
+
+	    tbytes= Grid.StencilSendToRecvFrom((void *)&xbuf[dir][0], xmit_to_rank,
+					       (void *)&rbuf[dir][0], recv_from_rank, bytes,dir);
 
 #pragma omp atomic
 	    dbytes+=tbytes;
diff --git a/lib/allocator/AlignedAllocator.h b/lib/allocator/AlignedAllocator.h
index 3609d8ab..c5ad0883 100644
--- a/lib/allocator/AlignedAllocator.h
+++ b/lib/allocator/AlignedAllocator.h
@@ -92,6 +92,9 @@ public:
     size_type bytes = __n*sizeof(_Tp);
 
     _Tp *ptr = (_Tp *) PointerCache::Lookup(bytes);
+    //    if ( ptr != NULL ) 
+    //      std::cout << "alignedAllocator "<<__n << " cache hit "<< std::hex << ptr <<std::dec <<std::endl;
+
     //////////////////
     // Hack 2MB align; could make option probably doesn't need configurability
     //////////////////
@@ -102,6 +105,7 @@ public:
 #else
     if ( ptr == (_Tp *) NULL ) ptr = (_Tp *) memalign(GRID_ALLOC_ALIGN,bytes);
 #endif
+    //    std::cout << "alignedAllocator " << std::hex << ptr <<std::dec <<std::endl;
     // First touch optimise in threaded loop
     uint8_t *cp = (uint8_t *)ptr;
 #ifdef GRID_OMP
@@ -115,6 +119,7 @@ public:
 
   void deallocate(pointer __p, size_type __n) { 
     size_type bytes = __n * sizeof(_Tp);
+
     pointer __freeme = (pointer)PointerCache::Insert((void *)__p,bytes);
 
 #ifdef HAVE_MM_MALLOC_H
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 3378c56a..956de0d2 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -153,8 +153,10 @@ void CartesianCommunicator::ShmInitGeneric(void){
   if ( Hugepages ) mmap_flag |= MAP_HUGETLB;
 #endif
   ShmCommBuf =(void *) mmap(NULL, MAX_MPI_SHM_BYTES, PROT_READ | PROT_WRITE, mmap_flag, -1, 0); 
-  if (ShmCommBuf == (void *)MAP_FAILED) exit(EXIT_FAILURE);  
-  std::cout << "ShmCommBuf "<<ShmCommBuf<<std::endl;
+  if (ShmCommBuf == (void *)MAP_FAILED) {
+    perror("mmap failed ");
+    exit(EXIT_FAILURE);  
+  }
 #else 
   ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES);
   ShmCommBuf=(void *)&ShmBufStorageVector[0];
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 204993fd..cb7fa390 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -221,8 +221,9 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
       if ( ptr == MAP_FAILED ) {       perror("failed mmap");      assert(0);    }
       assert(((uint64_t)ptr&0x3F)==0);
 
-      // Try to force numa domain on the shm segment if we have numaif.h
-#ifdef HAVE_NUMAIF_H
+// Experiments; Experiments; Try to force numa domain on the shm segment if we have numaif.h
+#if 0
+//#ifdef HAVE_NUMAIF_H
 	int status;
 	int flags=MPOL_MF_MOVE;
 #ifdef KNL
diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc
index 9a9b26d2..eb6ef87d 100644
--- a/lib/communicator/Communicator_mpit.cc
+++ b/lib/communicator/Communicator_mpit.cc
@@ -242,11 +242,24 @@ double CartesianCommunicator::StencilSendToRecvFromBegin(std::vector<CommsReques
 							 int recv_from_rank,
 							 int bytes,int dir)
 {
-  StencilSendToRecvFrom(xmit,xmit_to_rank,recv,recv_from_rank,bytes,dir);
+  int myrank = _processor;
+  int ierr;
+  assert(dir < communicator_halo.size());
+  
+  //  std::cout << " sending on communicator "<<dir<<" " <<communicator_halo[dir]<<std::endl;
+  // Give the CPU to MPI immediately; can use threads to overlap optionally
+  MPI_Request req[2];
+  MPI_Irecv(recv,bytes,MPI_CHAR,recv_from_rank,recv_from_rank, communicator_halo[dir],&req[1]);
+  MPI_Isend(xmit,bytes,MPI_CHAR,xmit_to_rank  ,myrank        , communicator_halo[dir],&req[0]);
+
+  list.push_back(req[0]);
+  list.push_back(req[1]);
+  return 2.0*bytes;
 }
 void CartesianCommunicator::StencilSendToRecvFromComplete(std::vector<CommsRequest_t> &waitall,int dir)
 { 
-  // Do nothing
+  int nreq=waitall.size();
+  MPI_Waitall(nreq, &waitall[0], MPI_STATUSES_IGNORE);
 };
 double CartesianCommunicator::StencilSendToRecvFrom(void *xmit,
 						    int xmit_to_rank,
@@ -262,7 +275,7 @@ double CartesianCommunicator::StencilSendToRecvFrom(void *xmit,
   // Give the CPU to MPI immediately; can use threads to overlap optionally
   MPI_Request req[2];
   MPI_Irecv(recv,bytes,MPI_CHAR,recv_from_rank,recv_from_rank, communicator_halo[dir],&req[1]);
-  MPI_Isend(xmit,bytes,MPI_CHAR,xmit_to_rank,myrank,          communicator_halo[dir], &req[0]);
+  MPI_Isend(xmit,bytes,MPI_CHAR,xmit_to_rank  ,myrank        , communicator_halo[dir],&req[0]);
   MPI_Waitall(2, req, MPI_STATUSES_IGNORE);
   return 2.0*bytes;
 }
diff --git a/lib/qcd/action/fermion/CayleyFermion5D.cc b/lib/qcd/action/fermion/CayleyFermion5D.cc
index 46ba3793..5e67d1f1 100644
--- a/lib/qcd/action/fermion/CayleyFermion5D.cc
+++ b/lib/qcd/action/fermion/CayleyFermion5D.cc
@@ -429,7 +429,7 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   
   for(int i=0;i<Ls;i++){
     bee[i]=as[i]*(bs[i]*(4.0-this->M5) +1.0);     
-    //    assert(fabs(bee[i])>0.0);
+    assert(fabs(bee[i])>0.0);
     cee[i]=as[i]*(1.0-cs[i]*(4.0-this->M5));
     beo[i]=as[i]*bs[i];
     ceo[i]=-as[i]*cs[i];
@@ -455,11 +455,17 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
     dee[i] = bee[i];
     
     if ( i < Ls-1 ) {
+
+      assert(fabs(bee[i])>0.0);
+      assert(fabs(bee[0])>0.0);
       
       lee[i] =-cee[i+1]/bee[i]; // sub-diag entry on the ith column
       
       leem[i]=mass*cee[Ls-1]/bee[0];
-      for(int j=0;j<i;j++)  leem[i]*= aee[j]/bee[j+1];
+      for(int j=0;j<i;j++) {
+	assert(fabs(bee[j+1])>0.0);
+	leem[i]*= aee[j]/bee[j+1];
+      }
       
       uee[i] =-aee[i]/bee[i];   // up-diag entry on the ith row
       
@@ -478,7 +484,7 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   { 
     Coeff_t delta_d=mass*cee[Ls-1];
     for(int j=0;j<Ls-1;j++) {
-      //      assert(fabs(bee[j])>0.0);
+      assert(fabs(bee[j])>0.0);
       delta_d *= cee[j]/bee[j];
     }
     dee[Ls-1] += delta_d;
diff --git a/lib/qcd/action/fermion/WilsonCompressor.h b/lib/qcd/action/fermion/WilsonCompressor.h
index 96cbe1ec..30c6d838 100644
--- a/lib/qcd/action/fermion/WilsonCompressor.h
+++ b/lib/qcd/action/fermion/WilsonCompressor.h
@@ -238,7 +238,35 @@ template<typename HCS,typename HS,typename S> using WilsonCompressor = WilsonCom
 template<class vobj,class cobj>
 class WilsonStencil : public CartesianStencil<vobj,cobj> {
 public:
-
+  double timer0;
+  double timer1;
+  double timer2;
+  double timer3;
+  double timer4;
+  double timer5;
+  double timer6;
+  uint64_t callsi;
+  void ZeroCountersi(void)
+  {
+    std::cout << GridLogMessage << " ZeroCountersi()"<<std::endl;
+    timer0=0;
+    timer1=0;
+    timer2=0;
+    timer3=0;
+    timer4=0;
+    timer5=0;
+    timer6=0;
+    callsi=0;
+  }
+  void Reporti(int calls)
+  {
+    std::cout << GridLogMessage << " Reporti() calls " <<callsi << calls<<std::endl;
+    if ( timer0 ) std::cout << GridLogMessage << " timer0 (HaloGatherOpt) " <<timer0/calls <<std::endl;
+    if ( timer1 ) std::cout << GridLogMessage << " timer1 (Communicate)   " <<timer1/calls <<std::endl;
+    if ( timer2 ) std::cout << GridLogMessage << " timer2 (CommsMerge )   " <<timer2/calls <<std::endl;
+    if ( timer3 ) std::cout << GridLogMessage << " timer3 (commsMergeShm) " <<timer3/calls <<std::endl;
+    if ( timer4 ) std::cout << GridLogMessage << " timer4 " <<timer4 <<std::endl;
+  }
   typedef CartesianCommunicator::CommsRequest_t CommsRequest_t;
 
   std::vector<int> same_node;
@@ -252,6 +280,7 @@ public:
     : CartesianStencil<vobj,cobj> (grid,npoints,checkerboard,directions,distances) ,
     same_node(npoints)
   { 
+    ZeroCountersi();
     surface_list.resize(0);
   };
 
@@ -282,17 +311,25 @@ public:
   {
     std::vector<std::vector<CommsRequest_t> > reqs;
     this->HaloExchangeOptGather(source,compress);
+    double t1=usecond();
     this->CommunicateBegin(reqs);
     this->CommunicateComplete(reqs);
+    double t2=usecond(); timer1 += t2-t1;
     this->CommsMerge(compress);
+    double t3=usecond(); timer2 += t3-t2;
     this->CommsMergeSHM(compress);
+    double t4=usecond(); timer3 += t4-t3;
   }
   
   template <class compressor>
   void HaloExchangeOptGather(const Lattice<vobj> &source,compressor &compress) 
   {
     this->Prepare();
+    double t0=usecond();
     this->HaloGatherOpt(source,compress);
+    double t1=usecond();
+    timer0 += t1-t0;
+    callsi++;
   }
 
   template <class compressor>
@@ -304,7 +341,9 @@ public:
     typedef typename compressor::SiteHalfSpinor     SiteHalfSpinor;
     typedef typename compressor::SiteHalfCommSpinor SiteHalfCommSpinor;
 
+    this->mpi3synctime_g-=usecond();
     this->_grid->StencilBarrier();
+    this->mpi3synctime_g+=usecond();
 
     assert(source._grid==this->_grid);
     this->halogtime-=usecond();
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index 404ecce0..c5b0f872 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -185,6 +185,11 @@ void WilsonFermion5D<Impl>::Report(void)
     std::cout << GridLogMessage << "WilsonFermion5D StencilEven"<<std::endl;  StencilEven.Report();
     std::cout << GridLogMessage << "WilsonFermion5D StencilOdd" <<std::endl;  StencilOdd.Report();
   }
+  if ( DhopCalls > 0){
+    std::cout << GridLogMessage << "WilsonFermion5D Stencil     Reporti()"    <<std::endl;  Stencil.Reporti(DhopCalls);
+    std::cout << GridLogMessage << "WilsonFermion5D StencilEven Reporti()"<<std::endl;  StencilEven.Reporti(DhopCalls);
+    std::cout << GridLogMessage << "WilsonFermion5D StencilOdd  Reporti()" <<std::endl;  StencilOdd.Reporti(DhopCalls);
+  }
 }
 
 template<class Impl>
@@ -204,6 +209,9 @@ void WilsonFermion5D<Impl>::ZeroCounters(void) {
   Stencil.ZeroCounters();
   StencilEven.ZeroCounters();
   StencilOdd.ZeroCounters();
+  Stencil.ZeroCountersi();
+  StencilEven.ZeroCountersi();
+  StencilOdd.ZeroCountersi();
 }
 
 
@@ -445,6 +453,9 @@ void WilsonFermion5D<Impl>::DhopInternalOverlappedComms(StencilImpl & st, Lebesg
   DhopCommTime += ctime;
   DhopComputeTime+=ptime;
 
+  // First to enter, last to leave timing
+  st.CollateThreads();
+
   DhopFaceTime-=usecond();
   st.CommsMerge(compressor);
   DhopFaceTime+=usecond();
diff --git a/lib/stencil/Stencil.h b/lib/stencil/Stencil.h
index cca67587..ad454bcb 100644
--- a/lib/stencil/Stencil.h
+++ b/lib/stencil/Stencil.h
@@ -176,6 +176,9 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
   // Timing info; ugly; possibly temporary
   /////////////////////////////////////////
   double commtime;
+  double mpi3synctime;
+  double mpi3synctime_g;
+  double shmmergetime;
   double gathertime;
   double gathermtime;
   double halogtime;
@@ -185,8 +188,10 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
   double splicetime;
   double nosplicetime;
   double calls;
-  std::vector<double> comms_bytesthr;
-  std::vector<double> commtimethr;
+  std::vector<double> comm_bytes_thr;
+  std::vector<double> comm_time_thr;
+  std::vector<double> comm_enter_thr;
+  std::vector<double> comm_leave_thr;
 
   ////////////////////////////////////////
   // Stencil query
@@ -262,18 +267,45 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
 #endif
     if (nthreads == -1) nthreads = 1;
     if (mythread < nthreads) {
+      comm_enter_thr[mythread] = usecond();
       for (int i = mythread; i < Packets.size(); i += nthreads) {
-	double start = usecond();
 	uint64_t bytes = _grid->StencilSendToRecvFrom(Packets[i].send_buf,
 						      Packets[i].to_rank,
 						      Packets[i].recv_buf,
 						      Packets[i].from_rank,
 						      Packets[i].bytes,i);
-	comms_bytesthr[mythread] += bytes;
-	commtimethr[mythread] += usecond() - start;
+	comm_bytes_thr[mythread] += bytes;
       }
+      comm_leave_thr[mythread]= usecond();
+      comm_time_thr[mythread] += comm_leave_thr[mythread] - comm_enter_thr[mythread];
     }
   }
+  
+  void CollateThreads(void)
+  {
+    int nthreads = CartesianCommunicator::nCommThreads;
+    double first=0.0;
+    double last =0.0;
+
+    for(int t=0;t<nthreads;t++) {
+
+      double t0 = comm_enter_thr[t];
+      double t1 = comm_leave_thr[t];
+      comms_bytes+=comm_bytes_thr[t];
+
+      comm_enter_thr[t] = 0.0;
+      comm_leave_thr[t] = 0.0;
+      comm_time_thr[t]   = 0.0;
+      comm_bytes_thr[t]=0;
+
+      if ( first == 0.0 ) first = t0;                   // first is t0
+      if ( (t0 > 0.0) && ( t0 < first ) ) first = t0;   // min time seen
+
+      if ( t1 > last ) last = t1;                       // max time seen
+      
+    }
+    commtime+= last-first;
+  }
   void CommunicateBegin(std::vector<std::vector<CommsRequest_t> > &reqs)
   {
     reqs.resize(Packets.size());
@@ -295,14 +327,48 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
     }
     commtime+=usecond();
   }
+  void Communicate(void)
+  {
+#ifdef GRID_OMP
+#pragma omp parallel 
+    {
+      // must be called in parallel region
+      int mythread  = omp_get_thread_num();
+      int maxthreads= omp_get_max_threads();
+      int nthreads = CartesianCommunicator::nCommThreads;
+      assert(nthreads <= maxthreads);
+
+      if (nthreads == -1) nthreads = 1;
+#else
+      int mythread = 0;
+      int nthreads = 1;
+#endif
+      if (mythread < nthreads) {
+	for (int i = mythread; i < Packets.size(); i += nthreads) {
+	  double start = usecond();
+	  comm_bytes_thr[mythread] += _grid->StencilSendToRecvFrom(Packets[i].send_buf,
+								   Packets[i].to_rank,
+								   Packets[i].recv_buf,
+								   Packets[i].from_rank,
+								   Packets[i].bytes,i);
+	  comm_time_thr[mythread] += usecond() - start;
+	}
+      }
+#ifdef GRID_OMP
+    }
+#endif
+  }
   
   template<class compressor> void HaloExchange(const Lattice<vobj> &source,compressor &compress) 
   {
     std::vector<std::vector<CommsRequest_t> > reqs;
     Prepare();
     HaloGather(source,compress);
+    // Concurrent
     CommunicateBegin(reqs);
     CommunicateComplete(reqs);
+    // Sequential
+    // Communicate();
     CommsMergeSHM(compress); 
     CommsMerge(compress); 
   }
@@ -363,7 +429,9 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
   template<class compressor>
   void HaloGather(const Lattice<vobj> &source,compressor &compress)
   {
+    mpi3synctime_g-=usecond();
     _grid->StencilBarrier();// Synch shared memory on a single nodes
+    mpi3synctime_g+=usecond();
 
     // conformable(source._grid,_grid);
     assert(source._grid==_grid);
@@ -423,8 +491,12 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
     CommsMerge(decompress,Mergers,Decompressions); 
   }
   template<class decompressor>  void CommsMergeSHM(decompressor decompress) {
+    mpi3synctime-=usecond();    
     _grid->StencilBarrier();// Synch shared memory on a single nodes
+    mpi3synctime+=usecond();    
+    shmmergetime-=usecond();    
     CommsMerge(decompress,MergersSHM,DecompressionsSHM);
+    shmmergetime+=usecond();    
   }
 
   template<class decompressor>
@@ -470,8 +542,10 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
 		  const std::vector<int> &distances) 
    : _permute_type(npoints), 
     _comm_buf_size(npoints),
-    comms_bytesthr(npoints), 
-       commtimethr(npoints)
+    comm_bytes_thr(npoints), 
+    comm_enter_thr(npoints),
+    comm_leave_thr(npoints), 
+       comm_time_thr(npoints)
   {
     face_table_computed=0;
     _npoints = npoints;
@@ -1025,8 +1099,15 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
   void ZeroCounters(void) {
     gathertime = 0.;
     commtime = 0.;
-    memset(&commtimethr[0], 0, sizeof(commtimethr));
-    memset(&comms_bytesthr[0], 0, sizeof(comms_bytesthr));
+    mpi3synctime=0.;
+    mpi3synctime_g=0.;
+    shmmergetime=0.;
+    for(int i=0;i<_npoints;i++){
+      comm_time_thr[i]=0;
+      comm_bytes_thr[i]=0;
+      comm_enter_thr[i]=0;
+      comm_leave_thr[i]=0;
+    }
     halogtime = 0.;
     mergetime = 0.;
     decompresstime = 0.;
@@ -1043,13 +1124,17 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
     RealD NP = _grid->_Nprocessors;
     RealD NN = _grid->NodeCount();
     double t = 0;
-    // if commtimethr is set they were all done in parallel so take the max
+    // if comm_time_thr is set they were all done in parallel so take the max
     // but add up the bytes
+    int threaded = 0 ;
     for (int i = 0; i < 8; ++i) {
-      comms_bytes += comms_bytesthr[i];
-      if (t < commtimethr[i]) t = commtimethr[i];
+      if ( comm_time_thr[i]>0.0 ) {
+	threaded = 1;
+	comms_bytes += comm_bytes_thr[i];
+	if (t < comm_time_thr[i]) t = comm_time_thr[i];
+      }
     }
-    commtime += t;
+    if (threaded) commtime += t;
     
     _grid->GlobalSum(commtime);    commtime/=NP;
     if ( calls > 0. ) {
@@ -1065,6 +1150,9 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
 	std::cout << GridLogMessage << " Stencil " << comms_bytes/commtime/1000. << " GB/s per rank"<<std::endl;
 	std::cout << GridLogMessage << " Stencil " << comms_bytes/commtime/1000.*NP/NN << " GB/s per node"<<std::endl;
       }
+      PRINTIT(mpi3synctime);
+      PRINTIT(mpi3synctime_g);
+      PRINTIT(shmmergetime);
       PRINTIT(splicetime);
       PRINTIT(nosplicetime);
     }

From c289699d9a4a064a3770b6fca6c4ecf6c55f3553 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Fri, 25 Aug 2017 11:41:01 +0100
Subject: [PATCH 070/115] updated from cambridge mpi3 shakeout

---
 benchmarks/Benchmark_ITT.cc               | 4 ++--
 lib/qcd/action/fermion/WilsonCompressor.h | 7 +++++--
 lib/stencil/Stencil.h                     | 8 ++++----
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index c5226ee1..bd75dd8e 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -108,7 +108,7 @@ public:
 
   static void Comms(void)
   {
-    int Nloop=1000;
+    int Nloop=200;
     int nmu=0;
     int maxlat=32;
 
@@ -197,7 +197,7 @@ public:
 
 	timestat.statistics(t_time);
 	//	for(int i=0;i<t_time.size();i++){
-	  //	  std::cout << i<<" "<<t_time[i]<<std::endl;
+	//	  std::cout << i<<" "<<t_time[i]<<std::endl;
 	//	}
 
 	dbytes=dbytes*ppn;
diff --git a/lib/qcd/action/fermion/WilsonCompressor.h b/lib/qcd/action/fermion/WilsonCompressor.h
index 30c6d838..406476b0 100644
--- a/lib/qcd/action/fermion/WilsonCompressor.h
+++ b/lib/qcd/action/fermion/WilsonCompressor.h
@@ -312,8 +312,11 @@ public:
     std::vector<std::vector<CommsRequest_t> > reqs;
     this->HaloExchangeOptGather(source,compress);
     double t1=usecond();
-    this->CommunicateBegin(reqs);
-    this->CommunicateComplete(reqs);
+    // Asynchronous MPI calls multidirectional, Isend etc...
+    //    this->CommunicateBegin(reqs);
+    //    this->CommunicateComplete(reqs);
+    // Non-overlapped directions within a thread. Asynchronous calls except MPI3, threaded up to comm threads ways.
+    this->Communicate();
     double t2=usecond(); timer1 += t2-t1;
     this->CommsMerge(compress);
     double t3=usecond(); timer2 += t3-t2;
diff --git a/lib/stencil/Stencil.h b/lib/stencil/Stencil.h
index ad454bcb..cd0792d5 100644
--- a/lib/stencil/Stencil.h
+++ b/lib/stencil/Stencil.h
@@ -365,10 +365,10 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
     Prepare();
     HaloGather(source,compress);
     // Concurrent
-    CommunicateBegin(reqs);
-    CommunicateComplete(reqs);
-    // Sequential
-    // Communicate();
+    //CommunicateBegin(reqs);
+    //CommunicateComplete(reqs);
+    // Sequential, possibly threaded
+    Communicate();
     CommsMergeSHM(compress); 
     CommsMerge(compress); 
   }

From 3a582174053732f4e5645367b750fd446d8fcb1d Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Fri, 25 Aug 2017 14:29:53 +0100
Subject: [PATCH 071/115] Updated

---
 benchmarks/Benchmark_ITT.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index bd75dd8e..2edae8d0 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -386,7 +386,7 @@ public:
 	if ( sizeof(Real)==8 )   std::cout << GridLogMessage<< "* DOUBLE precision "<<std::endl;
 	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
-	int nwarm = 200;
+	int nwarm = 100;
 	double t0=usecond();
 	sFGrid->Barrier();
 	for(int i=0;i<nwarm;i++){
@@ -396,7 +396,7 @@ public:
 	double t1=usecond();
 	//	uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0);
 	//	if (ncall < 500) ncall = 500;
-	uint64_t ncall = 1000;
+	uint64_t ncall = 500;
 
 	sFGrid->Broadcast(0,&ncall,sizeof(ncall));
 

From f365a83fae6279ca1c52d90d4d266fc39f46335f Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Fri, 25 Aug 2017 14:24:11 -0400
Subject: [PATCH 072/115] In G-parity unrolled kernel, replaced calls to
 permute and exchange with run-time-evaluated permute type with explicit calls
 to appropriate underlying functions

---
 lib/qcd/action/fermion/WilsonKernelsHand.cc | 138 +++++++++-----------
 lib/simd/Grid_vector_types.h                |  13 +-
 2 files changed, 77 insertions(+), 74 deletions(-)

diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index ca93916b..a0f5ffec 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -44,7 +44,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   Chimu_31=ref(F)(3)(1);			\
   Chimu_32=ref(F)(3)(2)
 
-#define LOAD_CHIMU(DIR,F)						\
+#define LOAD_CHIMU(DIR,F,PERM)						\
   { const SiteSpinor & ref (in._odata[offset]); LOAD_CHIMU_BODY(F); }
 
 #define LOAD_CHI_BODY(F)				\
@@ -55,7 +55,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     Chi_11 = ref(F)(1)(1);\
     Chi_12 = ref(F)(1)(2)
 
-#define LOAD_CHI(DIR,F)						\
+#define LOAD_CHI(DIR,F,PERM)					\
   {const SiteHalfSpinor &ref(buf[offset]); LOAD_CHI_BODY(F); }
 
 
@@ -73,33 +73,27 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 
 //1l 1h -> 1h 1l
 //0l 0h , 1h 1l -> 0l 1h 0h,1l
-#define DO_TWIST_0L_1H(INTO,S,C,F, tmp1, tmp2, tmp3) \
-  permute(tmp1, ref(1)(S)(C), permute_type);				\
-  exchange(tmp2,tmp3, ref(0)(S)(C), tmp1, permute_type);		\
+#define DO_TWIST_0L_1H(INTO,S,C,F, PERM, tmp1, tmp2, tmp3)			\
+  permute##PERM(tmp1, ref(1)(S)(C));				\
+  exchange##PERM(tmp2,tmp3, ref(0)(S)(C), tmp1);		\
   INTO = tmp2;
 
 //0l 0h -> 0h 0l
 //1l 1h, 0h 0l -> 1l 0h, 1h 0l
-#define DO_TWIST_1L_0H(INTO,S,C,F, tmp1, tmp2, tmp3)			\
-  permute(tmp1, ref(0)(S)(C), permute_type);				\
-  exchange(tmp2,tmp3, ref(1)(S)(C), tmp1, permute_type);		\
+#define DO_TWIST_1L_0H(INTO,S,C,F, PERM, tmp1, tmp2, tmp3)			\
+  permute##PERM(tmp1, ref(0)(S)(C));				\
+  exchange##PERM(tmp2,tmp3, ref(1)(S)(C), tmp1);		\
   INTO = tmp2;
 
-#define DO_TWIST(INTO,S,C,F, tmp1, tmp2, tmp3)				\
-  if(  ( F==0 && ((distance == 1 && !perm) || (distance == -1 && perm)) ) || \
-       ( F==1 && ((distance == -1 && !perm) || (distance == 1 && perm)) ) ){ \
-    DO_TWIST_0L_1H(INTO,S,C,F,tmp1,tmp2,tmp3);				\
-  }else{								\
-    DO_TWIST_1L_0H(INTO,S,C,F,tmp1,tmp2,tmp3);				\
-  }
+
 
 
 #define LOAD_CHI_SETUP(DIR,F)						\
-  int g = F;								\
-  const int direction = st._directions[DIR];				\
-  const int distance = st._distances[DIR];				\
-  const int sl = st._grid->_simd_layout[direction];			\
-  int inplace_twist = 0;						\
+  g = F;								\
+  direction = st._directions[DIR];				\
+  distance = st._distances[DIR];				\
+  sl = st._grid->_simd_layout[direction];			\
+  inplace_twist = 0;						\
   if(SE->_around_the_world && this->Params.twists[DIR % 4]){		\
     if(sl == 1){							\
       g = (F+1) % 2;							\
@@ -108,74 +102,72 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
     }									\
   }  
 
-#define LOAD_CHIMU_GPARITY_INPLACE_TWIST(DIR,F)				\
+#define LOAD_CHIMU_GPARITY_INPLACE_TWIST(DIR,F,PERM)			\
   { const SiteSpinor &ref(in._odata[offset]);				\
     LOAD_CHI_SETUP(DIR,F);						\
     if(!inplace_twist){							\
       LOAD_CHIMU_BODY(g);						\
     }else{								\
-      const int permute_type = st._grid->PermuteType(direction);	\
       if(  ( F==0 && ((distance == 1 && !perm) || (distance == -1 && perm)) ) || \
 	   ( F==1 && ((distance == -1 && !perm) || (distance == 1 && perm)) ) ){ \
-	DO_TWIST_0L_1H(Chimu_00,0,0,F,  U_00,U_01,U_10);		\
-	DO_TWIST_0L_1H(Chimu_01,0,1,F,  U_11,U_20,U_21);		\
-	DO_TWIST_0L_1H(Chimu_02,0,2,F,  U_00,U_01,U_10);		\
-	DO_TWIST_0L_1H(Chimu_10,1,0,F,  U_11,U_20,U_21);		\
-	DO_TWIST_0L_1H(Chimu_11,1,1,F,  U_00,U_01,U_10);		\
-	DO_TWIST_0L_1H(Chimu_12,1,2,F,  U_11,U_20,U_21);		\
-	DO_TWIST_0L_1H(Chimu_20,2,0,F,  U_00,U_01,U_10);		\
-	DO_TWIST_0L_1H(Chimu_21,2,1,F,  U_11,U_20,U_21);		\
-	DO_TWIST_0L_1H(Chimu_22,2,2,F,  U_00,U_01,U_10);		\
-	DO_TWIST_0L_1H(Chimu_30,3,0,F,  U_11,U_20,U_21);		\
-	DO_TWIST_0L_1H(Chimu_31,3,1,F,  U_00,U_01,U_10);		\
-	DO_TWIST_0L_1H(Chimu_32,3,2,F,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_00,0,0,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_01,0,1,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_02,0,2,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_10,1,0,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_11,1,1,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_12,1,2,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_20,2,0,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_21,2,1,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_22,2,2,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_30,3,0,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_0L_1H(Chimu_31,3,1,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_0L_1H(Chimu_32,3,2,F,PERM,  U_11,U_20,U_21);		\
       }else{								\
-	DO_TWIST_1L_0H(Chimu_00,0,0,F,  U_00,U_01,U_10);		\
-	DO_TWIST_1L_0H(Chimu_01,0,1,F,  U_11,U_20,U_21);		\
-	DO_TWIST_1L_0H(Chimu_02,0,2,F,  U_00,U_01,U_10);		\
-	DO_TWIST_1L_0H(Chimu_10,1,0,F,  U_11,U_20,U_21);		\
-	DO_TWIST_1L_0H(Chimu_11,1,1,F,  U_00,U_01,U_10);		\
-	DO_TWIST_1L_0H(Chimu_12,1,2,F,  U_11,U_20,U_21);		\
-	DO_TWIST_1L_0H(Chimu_20,2,0,F,  U_00,U_01,U_10);		\
-	DO_TWIST_1L_0H(Chimu_21,2,1,F,  U_11,U_20,U_21);		\
-	DO_TWIST_1L_0H(Chimu_22,2,2,F,  U_00,U_01,U_10);		\
-	DO_TWIST_1L_0H(Chimu_30,3,0,F,  U_11,U_20,U_21);		\
-	DO_TWIST_1L_0H(Chimu_31,3,1,F,  U_00,U_01,U_10);		\
-	DO_TWIST_1L_0H(Chimu_32,3,2,F,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_00,0,0,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_01,0,1,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_02,0,2,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_10,1,0,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_11,1,1,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_12,1,2,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_20,2,0,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_21,2,1,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_22,2,2,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_30,3,0,F,PERM,  U_11,U_20,U_21);		\
+	DO_TWIST_1L_0H(Chimu_31,3,1,F,PERM,  U_00,U_01,U_10);		\
+	DO_TWIST_1L_0H(Chimu_32,3,2,F,PERM,  U_11,U_20,U_21);		\
       } \
     } \
   }
 
 
-#define LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F)				\
+#define LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F,PERM)				\
   { const SiteHalfSpinor &ref(buf[offset]);				\
     LOAD_CHI_SETUP(DIR,F);						\
     if(!inplace_twist){							\
       LOAD_CHI_BODY(g);							\
     }else{								\
-      const int permute_type = st._grid->PermuteType(direction);	\
       if(  ( F==0 && ((distance == 1 && !perm) || (distance == -1 && perm)) ) || \
 	   ( F==1 && ((distance == -1 && !perm) || (distance == 1 && perm)) ) ){ \
-	DO_TWIST_0L_1H(Chi_00,0,0,F,  U_00,U_01,U_10);			\
-	DO_TWIST_0L_1H(Chi_01,0,1,F,  U_11,U_20,U_21);			\
-	DO_TWIST_0L_1H(Chi_02,0,2,F,  UChi_00,UChi_01,UChi_02);		\
-	DO_TWIST_0L_1H(Chi_10,1,0,F,  UChi_10,UChi_11,UChi_12);		\
-	DO_TWIST_0L_1H(Chi_11,1,1,F,  U_00,U_01,U_10);			\
-	DO_TWIST_0L_1H(Chi_12,1,2,F,  U_11,U_20,U_21);			\
+	DO_TWIST_0L_1H(Chi_00,0,0,F,PERM,  U_00,U_01,U_10);			\
+	DO_TWIST_0L_1H(Chi_01,0,1,F,PERM,  U_11,U_20,U_21);			\
+	DO_TWIST_0L_1H(Chi_02,0,2,F,PERM,  UChi_00,UChi_01,UChi_02);		\
+	DO_TWIST_0L_1H(Chi_10,1,0,F,PERM,  UChi_10,UChi_11,UChi_12);		\
+	DO_TWIST_0L_1H(Chi_11,1,1,F,PERM,  U_00,U_01,U_10);			\
+	DO_TWIST_0L_1H(Chi_12,1,2,F,PERM,  U_11,U_20,U_21);			\
       }else{								\
-	DO_TWIST_1L_0H(Chi_00,0,0,F,  U_00,U_01,U_10);			\
-	DO_TWIST_1L_0H(Chi_01,0,1,F,  U_11,U_20,U_21);			\
-	DO_TWIST_1L_0H(Chi_02,0,2,F,  UChi_00,UChi_01,UChi_02);		\
-	DO_TWIST_1L_0H(Chi_10,1,0,F,  UChi_10,UChi_11,UChi_12);		\
-	DO_TWIST_1L_0H(Chi_11,1,1,F,  U_00,U_01,U_10);			\
-	DO_TWIST_1L_0H(Chi_12,1,2,F,  U_11,U_20,U_21);			\
+	DO_TWIST_1L_0H(Chi_00,0,0,F,PERM,  U_00,U_01,U_10);			\
+	DO_TWIST_1L_0H(Chi_01,0,1,F,PERM,  U_11,U_20,U_21);			\
+	DO_TWIST_1L_0H(Chi_02,0,2,F,PERM,  UChi_00,UChi_01,UChi_02);		\
+	DO_TWIST_1L_0H(Chi_10,1,0,F,PERM,  UChi_10,UChi_11,UChi_12);		\
+	DO_TWIST_1L_0H(Chi_11,1,1,F,PERM,  U_00,U_01,U_10);			\
+	DO_TWIST_1L_0H(Chi_12,1,2,F,PERM,  U_11,U_20,U_21);			\
       }									\
     }									\
   }
 
 
-#define LOAD_CHI_GPARITY(DIR,F) LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F)
-#define LOAD_CHIMU_GPARITY(DIR,F) LOAD_CHIMU_GPARITY_INPLACE_TWIST(DIR,F)
+#define LOAD_CHI_GPARITY(DIR,F,PERM) LOAD_CHI_GPARITY_INPLACE_TWIST(DIR,F,PERM)
+#define LOAD_CHIMU_GPARITY(DIR,F,PERM) LOAD_CHIMU_GPARITY_INPLACE_TWIST(DIR,F,PERM)
 
 // To splat or not to splat depends on the implementation
 #define MULT_2SPIN_BODY \
@@ -442,13 +434,13 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   local  = SE->_is_local;			\
   perm   = SE->_permute;			\
   if ( local ) {				\
-    LOAD_CHIMU_IMPL(DIR,F);			\
+    LOAD_CHIMU_IMPL(DIR,F,PERM);			\
     PROJ;					\
     if ( perm) {				\
       PERMUTE_DIR(PERM);			\
     }						\
   } else {					\
-    LOAD_CHI_IMPL(DIR,F);			\
+    LOAD_CHI_IMPL(DIR,F,PERM);			\
   }						\
   MULT_2SPIN_IMPL(DIR,F);			\
   RECON;					
@@ -460,13 +452,13 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   local  = SE->_is_local;			\
   perm   = SE->_permute;			\
   if ( local ) {				\
-    LOAD_CHIMU_IMPL(DIR,F);			\
+    LOAD_CHIMU_IMPL(DIR,F,PERM);			\
     PROJ;					\
     if ( perm) {				\
       PERMUTE_DIR(PERM);			\
     }						\
   } else if ( st.same_node[DIR] ) {		\
-    LOAD_CHI_IMPL(DIR,F);			\
+    LOAD_CHI_IMPL(DIR,F,PERM);			\
   }						\
   if (local || st.same_node[DIR] ) {		\
     MULT_2SPIN_IMPL(DIR,F);			\
@@ -477,7 +469,7 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
   SE=st.GetEntry(ptype,DIR,ss);			\
   offset = SE->_offset;				\
   if((!SE->_is_local)&&(!st.same_node[DIR]) ) {	\
-    LOAD_CHI_IMPL(DIR,F);			\
+    LOAD_CHI_IMPL(DIR,F,PERM);			\
     MULT_2SPIN_IMPL(DIR,F);			\
     RECON;					\
     nmu++;					\
@@ -811,7 +803,7 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D
 									\
     HAND_DECLARATIONS(ignore);						\
 									\
-    int offset,local,perm, ptype;					\
+    int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \
     StencilEntry *SE;							\
     HAND_DOP_SITE(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
     HAND_DOP_SITE(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
@@ -828,7 +820,7 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D
     HAND_DECLARATIONS(ignore);						\
 									\
     StencilEntry *SE;							\
-    int offset,local,perm, ptype;					\
+    int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist;					\
     HAND_DOP_SITE_DAG(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
     HAND_DOP_SITE_DAG(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
   }									\
@@ -843,7 +835,7 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D
 									\
     HAND_DECLARATIONS(ignore);						\
 									\
-    int offset,local,perm, ptype;					\
+    int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist;					\
     StencilEntry *SE;							\
     HAND_DOP_SITE_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
     HAND_DOP_SITE_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
@@ -860,7 +852,7 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D
     HAND_DECLARATIONS(ignore);						\
 									\
     StencilEntry *SE;							\
-    int offset,local,perm, ptype;					\
+    int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \
     HAND_DOP_SITE_DAG_INT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
     HAND_DOP_SITE_DAG_INT(1, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
   }									\
@@ -875,7 +867,7 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D
 									\
     HAND_DECLARATIONS(ignore);						\
 									\
-    int offset,local,perm, ptype;					\
+    int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \
     StencilEntry *SE;							\
     int nmu=0;								\
     HAND_DOP_SITE_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
@@ -893,7 +885,7 @@ void WilsonKernels<Impl>::HandDhopSiteDagExt(StencilImpl &st,LebesgueOrder &lo,D
     HAND_DECLARATIONS(ignore);						\
 									\
     StencilEntry *SE;							\
-    int offset,local,perm, ptype;					\
+    int offset,local,perm, ptype, g, direction, distance, sl, inplace_twist; \
     int nmu=0;								\
     HAND_DOP_SITE_DAG_EXT(0, LOAD_CHI_GPARITY,LOAD_CHIMU_GPARITY,MULT_2SPIN_GPARITY); \
     nmu = 0;								\
diff --git a/lib/simd/Grid_vector_types.h b/lib/simd/Grid_vector_types.h
index 27585547..c67e74cb 100644
--- a/lib/simd/Grid_vector_types.h
+++ b/lib/simd/Grid_vector_types.h
@@ -376,7 +376,18 @@ class Grid_simd {
       Optimization::Exchange::Exchange0(out1.v,out2.v,in1.v,in2.v);
     }
   }
-
+  friend inline void exchange0(Grid_simd &out1,Grid_simd &out2,Grid_simd in1,Grid_simd in2){    
+    Optimization::Exchange::Exchange0(out1.v,out2.v,in1.v,in2.v);
+  }
+  friend inline void exchange1(Grid_simd &out1,Grid_simd &out2,Grid_simd in1,Grid_simd in2){    
+    Optimization::Exchange::Exchange1(out1.v,out2.v,in1.v,in2.v);
+  }
+  friend inline void exchange2(Grid_simd &out1,Grid_simd &out2,Grid_simd in1,Grid_simd in2){    
+    Optimization::Exchange::Exchange2(out1.v,out2.v,in1.v,in2.v);
+  }
+  friend inline void exchange3(Grid_simd &out1,Grid_simd &out2,Grid_simd in1,Grid_simd in2){    
+    Optimization::Exchange::Exchange3(out1.v,out2.v,in1.v,in2.v);
+  }
   ////////////////////////////////////////////////////////////////////
   // General permute; assumes vector length is same across
   // all subtypes; may not be a good assumption, but could

From d0f3d525d5dfb6cd7a2f5fe3be5a69c7ddc1306e Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Fri, 25 Aug 2017 19:33:54 +0100
Subject: [PATCH 073/115] Optimal block size for KNL

---
 benchmarks/Benchmark_ITT.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 2edae8d0..c0ce451f 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -679,8 +679,11 @@ int main (int argc, char ** argv)
   Grid_init(&argc,&argv);
 
   CartesianCommunicator::SetCommunicatorPolicy(CartesianCommunicator::CommunicatorPolicySequential);
+#ifdef KNL
+  LebesgueOrder::Block = std::vector<int>({8,2,2,2});
+#else
   LebesgueOrder::Block = std::vector<int>({2,2,2,2});
-
+#endif
   Benchmark::Decomposition();
 
   int do_memory=1;

From f68b5de9c8798779ef2657b9c2d469174ae8f53a Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 25 Aug 2017 19:35:21 +0100
Subject: [PATCH 074/115] No compile fix on Clang

---
 lib/qcd/action/fermion/CayleyFermion5D.cc | 12 ++++++------
 lib/qcd/action/fermion/WilsonCompressor.h |  4 ----
 lib/qcd/action/fermion/WilsonFermion5D.cc |  5 +++--
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/lib/qcd/action/fermion/CayleyFermion5D.cc b/lib/qcd/action/fermion/CayleyFermion5D.cc
index 5e67d1f1..838b1c3d 100644
--- a/lib/qcd/action/fermion/CayleyFermion5D.cc
+++ b/lib/qcd/action/fermion/CayleyFermion5D.cc
@@ -414,7 +414,7 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   for(int i=0; i < Ls; i++){
     as[i] = 1.0;
     omega[i] = gamma[i]*zolo_hi; //NB reciprocal relative to Chroma NEF code
-    //    assert(fabs(omega[i])>0.0);
+    assert(omega[i]!=Coeff_t(0.0));
     bs[i] = 0.5*(bpc/omega[i] + bmc);
     cs[i] = 0.5*(bpc/omega[i] - bmc);
   }
@@ -429,7 +429,7 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   
   for(int i=0;i<Ls;i++){
     bee[i]=as[i]*(bs[i]*(4.0-this->M5) +1.0);     
-    assert(fabs(bee[i])>0.0);
+    assert(bee[i]!=Coeff_t(0.0));
     cee[i]=as[i]*(1.0-cs[i]*(4.0-this->M5));
     beo[i]=as[i]*bs[i];
     ceo[i]=-as[i]*cs[i];
@@ -456,14 +456,14 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
     
     if ( i < Ls-1 ) {
 
-      assert(fabs(bee[i])>0.0);
-      assert(fabs(bee[0])>0.0);
+      assert(bee[i]!=Coeff_t(0.0));
+      assert(bee[0]!=Coeff_t(0.0));
       
       lee[i] =-cee[i+1]/bee[i]; // sub-diag entry on the ith column
       
       leem[i]=mass*cee[Ls-1]/bee[0];
       for(int j=0;j<i;j++) {
-	assert(fabs(bee[j+1])>0.0);
+	assert(bee[j+1]!=Coeff_t(0.0));
 	leem[i]*= aee[j]/bee[j+1];
       }
       
@@ -484,7 +484,7 @@ void CayleyFermion5D<Impl>::SetCoefficientsInternal(RealD zolo_hi,std::vector<Co
   { 
     Coeff_t delta_d=mass*cee[Ls-1];
     for(int j=0;j<Ls-1;j++) {
-      assert(fabs(bee[j])>0.0);
+      assert(bee[j] != Coeff_t(0.0));
       delta_d *= cee[j]/bee[j];
     }
     dee[Ls-1] += delta_d;
diff --git a/lib/qcd/action/fermion/WilsonCompressor.h b/lib/qcd/action/fermion/WilsonCompressor.h
index 406476b0..cc5c3c63 100644
--- a/lib/qcd/action/fermion/WilsonCompressor.h
+++ b/lib/qcd/action/fermion/WilsonCompressor.h
@@ -248,7 +248,6 @@ public:
   uint64_t callsi;
   void ZeroCountersi(void)
   {
-    std::cout << GridLogMessage << " ZeroCountersi()"<<std::endl;
     timer0=0;
     timer1=0;
     timer2=0;
@@ -260,7 +259,6 @@ public:
   }
   void Reporti(int calls)
   {
-    std::cout << GridLogMessage << " Reporti() calls " <<callsi << calls<<std::endl;
     if ( timer0 ) std::cout << GridLogMessage << " timer0 (HaloGatherOpt) " <<timer0/calls <<std::endl;
     if ( timer1 ) std::cout << GridLogMessage << " timer1 (Communicate)   " <<timer1/calls <<std::endl;
     if ( timer2 ) std::cout << GridLogMessage << " timer2 (CommsMerge )   " <<timer2/calls <<std::endl;
@@ -290,7 +288,6 @@ public:
     // Here we know the distance is 1 for WilsonStencil
     for(int point=0;point<this->_npoints;point++){
       same_node[point] = this->SameNode(point);
-      //      std::cout << " dir " <<point<<" same_node " <<same_node[point]<<std::endl;
     }
     
     for(int site = 0 ;site< vol4;site++){
@@ -365,7 +362,6 @@ public:
     int dag = compress.dag;
     int face_idx=0;
     if ( dag ) { 
-      //	std::cout << " Optimised Dagger compress " <<std::endl;
       assert(same_node[Xp]==this->HaloGatherDir(source,XpCompress,Xp,face_idx));
       assert(same_node[Yp]==this->HaloGatherDir(source,YpCompress,Yp,face_idx));
       assert(same_node[Zp]==this->HaloGatherDir(source,ZpCompress,Zp,face_idx));
diff --git a/lib/qcd/action/fermion/WilsonFermion5D.cc b/lib/qcd/action/fermion/WilsonFermion5D.cc
index c5b0f872..1da58ddb 100644
--- a/lib/qcd/action/fermion/WilsonFermion5D.cc
+++ b/lib/qcd/action/fermion/WilsonFermion5D.cc
@@ -123,12 +123,13 @@ WilsonFermion5D<Impl>::WilsonFermion5D(GaugeField &_Umu,
   int vol4;
   vol4=FourDimGrid.oSites();
   Stencil.BuildSurfaceList(LLs,vol4);
+
   vol4=FourDimRedBlackGrid.oSites();
   StencilEven.BuildSurfaceList(LLs,vol4);
    StencilOdd.BuildSurfaceList(LLs,vol4);
 
-  std::cout << GridLogMessage << " SurfaceLists "<< Stencil.surface_list.size()
-                       <<" " << StencilEven.surface_list.size()<<std::endl;
+   //  std::cout << GridLogMessage << " SurfaceLists "<< Stencil.surface_list.size()
+   //                       <<" " << StencilEven.surface_list.size()<<std::endl;
 
 }
      

From ad89abb018274ae29b372311e674070436e8bf9b Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 25 Aug 2017 20:43:37 +0100
Subject: [PATCH 075/115] Fix

---
 benchmarks/Benchmark_ITT.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 58fdb84a..c0ce451f 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -181,6 +181,7 @@ public:
 	  
 #ifdef GRID_OMP
 #pragma omp atomic
+#endif
 	      ncomm++;
 
 #ifdef GRID_OMP

From 54a5e6c1d0ec1cf1b66dac5ba407db49bc7e1016 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Fri, 25 Aug 2017 22:36:08 +0100
Subject: [PATCH 076/115] Check if we get huge pages on linux. Larry Meadows
 piece of magic.

---
 lib/allocator/AlignedAllocator.cc | 33 +++++++++++++++++++++++++++++++
 lib/allocator/AlignedAllocator.h  |  2 ++
 2 files changed, 35 insertions(+)

diff --git a/lib/allocator/AlignedAllocator.cc b/lib/allocator/AlignedAllocator.cc
index 04de20bf..764bd732 100644
--- a/lib/allocator/AlignedAllocator.cc
+++ b/lib/allocator/AlignedAllocator.cc
@@ -63,4 +63,37 @@ void *PointerCache::Lookup(size_t bytes) {
   return NULL;
 }
 
+
+void check_huge_pages(void *Buf,uint64_t BYTES)
+{
+#ifdef __linux__
+  int fd = open("/proc/self/pagemap", O_RDONLY);
+  assert(fd >= 0);
+  const int page_size = 4096;
+  uint64_t virt_pfn = (uint64_t)Buf / page_size;
+  off_t offset = sizeof(uint64_t) * virt_pfn;
+  uint64_t npages = (BYTES + page_size-1) / page_size;
+  uint64_t pagedata[npages];
+  uint64_t ret = lseek(fd, offset, SEEK_SET);
+  assert(ret == offset);
+  ret = ::read(fd, pagedata, sizeof(uint64_t)*npages);
+  assert(ret == sizeof(uint64_t) * npages);
+  int nhugepages = npages / 512;
+  int n4ktotal, nnothuge;
+  n4ktotal = 0;
+  nnothuge = 0;
+  for (int i = 0; i < nhugepages; ++i) {
+    uint64_t baseaddr = (pagedata[i*512] & 0x7fffffffffffffULL) * page_size;
+    for (int j = 0; j < 512; ++j) {
+      uint64_t pageaddr = (pagedata[i*512+j] & 0x7fffffffffffffULL) * page_size;
+      ++n4ktotal;
+      if (pageaddr != baseaddr + j * page_size)
+	++nnothuge;
+      }
+  }
+  int rank = CartesianCommunicator::RankWorld();
+  printf("rank %d Allocated %d 4k pages, %d not in huge pages\n", rank, n4ktotal, nnothuge);
+#endif
+}
+
 }
diff --git a/lib/allocator/AlignedAllocator.h b/lib/allocator/AlignedAllocator.h
index c5ad0883..e64a5949 100644
--- a/lib/allocator/AlignedAllocator.h
+++ b/lib/allocator/AlignedAllocator.h
@@ -64,6 +64,8 @@ namespace Grid {
 
   };
 
+  void check_huge_pages(void *Buf,uint64_t BYTES);
+
 ////////////////////////////////////////////////////////////////////
 // A lattice of something, but assume the something is SIMDized.
 ////////////////////////////////////////////////////////////////////

From 4b4c2a715b319bcc7060ef9ae8aa983c49471167 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sat, 26 Aug 2017 11:38:04 +0100
Subject: [PATCH 077/115] fcntl.h needed

---
 lib/allocator/AlignedAllocator.cc | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/allocator/AlignedAllocator.cc b/lib/allocator/AlignedAllocator.cc
index 764bd732..967b2571 100644
--- a/lib/allocator/AlignedAllocator.cc
+++ b/lib/allocator/AlignedAllocator.cc
@@ -1,7 +1,5 @@
-
-
-
 #include <Grid/GridCore.h>
+#include <fcntl.h>
 
 namespace Grid {
 

From 15fd4003efb86973cedd2c4a69740a33d214b9c9 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sun, 27 Aug 2017 13:46:02 +0100
Subject: [PATCH 078/115] Improving presentation of results

---
 benchmarks/Benchmark_ITT.cc | 58 ++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index c0ce451f..1f27a281 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -232,9 +232,13 @@ public:
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
     std::cout<<GridLogMessage << "= Benchmarking a*x + y bandwidth"<<std::endl;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-    std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<<std::endl;
+    std::cout<<GridLogMessage << "  L  "<<"\t\t"<<"bytes"<<"\t\t\t"<<"GB/s"<<"\t\t"<<"Gflop/s"<<"\t\t seconds"<< "\t\tGB/s / node"<<std::endl;
     std::cout<<GridLogMessage << "----------------------------------------------------------"<<std::endl;
   
+    uint64_t NP;
+    uint64_t NN;
+
+
   uint64_t lmax=48;
 #define NLOOP (100*lmax*lmax*lmax*lmax/lat/lat/lat/lat)
 
@@ -245,6 +249,9 @@ public:
       int64_t vol= latt_size[0]*latt_size[1]*latt_size[2]*latt_size[3];
       GridCartesian     Grid(latt_size,simd_layout,mpi_layout);
 
+      NP= Grid.RankCount();
+      NN =Grid.NodeCount();
+
       Vec rn ; random(sRNG,rn);
 
       LatticeVec z(&Grid); z=rn;
@@ -266,7 +273,8 @@ public:
       double flops=vol*Nvec*2;// mul,add
       double bytes=3.0*vol*Nvec*sizeof(Real);
       std::cout<<GridLogMessage<<std::setprecision(3) 
-	       << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.<<std::endl;
+	       << lat<<"\t\t"<<bytes<<"   \t\t"<<bytes/time<<"\t\t"<<flops/time<<"\t\t"<<(stop-start)/1000./1000.
+	       << "\t\t"<< bytes/time/NN <<std::endl;
 
     }
   };
@@ -387,6 +395,8 @@ public:
 	std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
 	int nwarm = 100;
+	uint64_t ncall = 1000;
+
 	double t0=usecond();
 	sFGrid->Barrier();
 	for(int i=0;i<nwarm;i++){
@@ -394,15 +404,8 @@ public:
 	}
 	sFGrid->Barrier();
 	double t1=usecond();
-	//	uint64_t ncall = (uint64_t) 2.5*1000.0*1000.0*nwarm/(t1-t0);
-	//	if (ncall < 500) ncall = 500;
-	uint64_t ncall = 500;
 
-	sFGrid->Broadcast(0,&ncall,sizeof(ncall));
-
-	//	std::cout << GridLogMessage << " Estimate " << ncall << " calls per second"<<std::endl;
 	sDw.ZeroCounters();
-
 	time_statistics timestat;
 	std::vector<double> t_time(ncall);
 	for(uint64_t i=0;i<ncall;i++){
@@ -692,26 +695,13 @@ int main (int argc, char ** argv)
   int do_wilson=1;
   int do_dwf   =1;
 
-  if ( do_memory ) {
-    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-    std::cout<<GridLogMessage << " Memory benchmark " <<std::endl;
-    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-    Benchmark::Memory();
-  }
-
-  if ( do_comms ) {
-    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-    std::cout<<GridLogMessage << " Communications benchmark " <<std::endl;
-    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-    Benchmark::Comms();
-  }
-
   if ( do_su3 ) {
     // empty for now
   }
 
   int sel=2;
   std::vector<int> L_list({8,12,16,24});
+  //  std::vector<int> L_list({8,12});
   std::vector<double> wilson;
   std::vector<double> dwf4;
   std::vector<double> dwf5;
@@ -744,6 +734,10 @@ int main (int argc, char ** argv)
       dwf5.push_back(Benchmark::DWF5(Ls,L_list[l]));
     }
 
+  }
+
+  if ( do_dwf ) {
+
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
   std::cout<<GridLogMessage << " Summary table Ls="<<Ls <<std::endl;
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
@@ -752,7 +746,24 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << L_list[l] <<" \t\t "<< wilson[l]<<" \t "<<dwf4[l]<<" \t "<<dwf5[l] <<std::endl;
   }
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+  }
 
+  if ( do_memory ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Memory benchmark " <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::Memory();
+  }
+
+  if ( do_comms ) {
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    std::cout<<GridLogMessage << " Communications benchmark " <<std::endl;
+    std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
+    Benchmark::Comms();
+  }
+
+
+  if ( do_dwf ) {
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
   std::cout<<GridLogMessage << " Per Node Summary table Ls="<<Ls <<std::endl;
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
@@ -767,7 +778,6 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << " Comparison point result: "  << dwf4[sel]/NN <<std::endl;
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
-
   }
 
 

From 5b9267e88d2e900446d43dc18965ee1083d86d35 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Sun, 27 Aug 2017 18:24:48 -0400
Subject: [PATCH 079/115] Cleaner comms benchmark treatment for one node runs

---
 benchmarks/Benchmark_ITT.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 1f27a281..73d5e7b8 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -748,6 +748,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
   }
 
+  int NN=NN_global;
   if ( do_memory ) {
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
     std::cout<<GridLogMessage << " Memory benchmark " <<std::endl;
@@ -755,7 +756,7 @@ int main (int argc, char ** argv)
     Benchmark::Memory();
   }
 
-  if ( do_comms ) {
+  if ( do_comms && (NN>1) ) {
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
     std::cout<<GridLogMessage << " Communications benchmark " <<std::endl;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
@@ -767,7 +768,6 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
   std::cout<<GridLogMessage << " Per Node Summary table Ls="<<Ls <<std::endl;
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-  int NN=NN_global;
   std::cout<<GridLogMessage << " L \t\t Wilson\t\t DWF4  \t\t DWF5 " <<std::endl;
   for(int l=0;l<L_list.size();l++){
     std::cout<<GridLogMessage << L_list[l] <<" \t\t "<< wilson[l]/NN<<" \t "<<dwf4[l]/NN<<" \t "<<dwf5[l] /NN<<std::endl;

From d36d2fb40d1e2871db5f7be4d79e58811037ad5d Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Mon, 28 Aug 2017 06:53:56 -0700
Subject: [PATCH 080/115] Added ability to override default Ls in Benchmark_dwf

---
 benchmarks/Benchmark_dwf.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/benchmarks/Benchmark_dwf.cc b/benchmarks/Benchmark_dwf.cc
index 7814ec7d..1f9c7624 100644
--- a/benchmarks/Benchmark_dwf.cc
+++ b/benchmarks/Benchmark_dwf.cc
@@ -51,7 +51,13 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "Grid is setup to use "<<threads<<" threads"<<std::endl;
 
   std::vector<int> latt4 = GridDefaultLatt();
-  const int Ls=16;
+  int Ls=16;
+  for(int i=0;i<argc;i++)
+    if(std::string(argv[i]) == "-Ls"){
+      std::stringstream ss(argv[i+1]); ss >> Ls;
+    }
+
+
   GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
   GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);

From 74af885d4eda81453b1eb83b062a11810011d5f5 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Tue, 29 Aug 2017 09:50:37 -0400
Subject: [PATCH 081/115] Removed some no-longer-needed associated with
 G-parity hand unrolled kernel

---
 lib/qcd/action/fermion/FermionOperatorImpl.h | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/lib/qcd/action/fermion/FermionOperatorImpl.h b/lib/qcd/action/fermion/FermionOperatorImpl.h
index ffb82989..9d24deb2 100644
--- a/lib/qcd/action/fermion/FermionOperatorImpl.h
+++ b/lib/qcd/action/fermion/FermionOperatorImpl.h
@@ -425,22 +425,6 @@ class DomainWallVec5dImpl :  public PeriodicGaugeImpl< GaugeImplTypes< S,Nrepres
     ////////////////////////////////////////////////////////////////////////////////////////
     // Flavour doubled spinors; is Gparity the only? what about C*?
     ////////////////////////////////////////////////////////////////////////////////////////
-namespace GparityWilsonImpl_helper{
-  template<typename Get, typename A,typename B>
-  struct getAB;
-  
-  template<typename A,typename B>
-  struct getAB<A,A,B>{
-    static inline A & ref(A &a, B &b){ return a; }
-  };
-  template<typename A,typename B>
-  struct getAB<B,A,B>{
-    static inline B & ref(A &a, B &b){ return b; }
-  };
-};
-  
-
-  
 template <class S, int Nrepresentation, class Options=CoeffReal>
 class GparityWilsonImpl : public ConjugateGaugeImpl<GaugeImplTypes<S, Nrepresentation> > {
  public:

From 59bd1fe21b3fc81b8c50e9caa53f2adc65b3d7b5 Mon Sep 17 00:00:00 2001
From: Christopher Kelly <ckelly@phys.columbia.edu>
Date: Tue, 29 Aug 2017 13:07:37 -0700
Subject: [PATCH 082/115] Fix for 'perm' and 'local' not being set for
 hand-unrolled external-site Dslash, which caused incorrect behavior of
 G-parity kernel

---
 lib/qcd/action/fermion/WilsonKernelsHand.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/qcd/action/fermion/WilsonKernelsHand.cc b/lib/qcd/action/fermion/WilsonKernelsHand.cc
index a0f5ffec..80b81714 100644
--- a/lib/qcd/action/fermion/WilsonKernelsHand.cc
+++ b/lib/qcd/action/fermion/WilsonKernelsHand.cc
@@ -468,6 +468,8 @@ Author: paboyle <paboyle@ph.ed.ac.uk>
 #define HAND_STENCIL_LEG_EXT(PROJ,PERM,DIR,RECON,F,LOAD_CHI_IMPL,LOAD_CHIMU_IMPL,MULT_2SPIN_IMPL)	\
   SE=st.GetEntry(ptype,DIR,ss);			\
   offset = SE->_offset;				\
+  local  = SE->_is_local;			\
+  perm   = SE->_permute;			\
   if((!SE->_is_local)&&(!st.same_node[DIR]) ) {	\
     LOAD_CHI_IMPL(DIR,F,PERM);			\
     MULT_2SPIN_IMPL(DIR,F);			\

From 7359df35015802c52e74ba47fa6791a65d441e17 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 31 Aug 2017 10:42:35 +0100
Subject: [PATCH 083/115] Full reporting for benchmark; save robustness factor

---
 benchmarks/Benchmark_ITT.cc           | 28 +++++++++++++++++++--------
 lib/allocator/AlignedAllocator.h      |  6 ++++--
 lib/communicator/Communicator_base.cc |  2 +-
 lib/util/Init.cc                      |  3 ++-
 4 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 73d5e7b8..3b064e19 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -439,12 +439,14 @@ public:
 	sDw.Report();
 
       }
+      double robust = mflops_worst/mflops_best;;
       std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
       std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " sDeo Best  mflop/s        =   "<< mflops_best << " ; " << mflops_best/NN<<" per node " <<std::endl;
       std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " sDeo Worst mflop/s        =   "<< mflops_worst<< " ; " << mflops_worst/NN<<" per node " <<std::endl;
-      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< mflops_worst/mflops_best <<std::endl;
+
+      std::cout<<GridLogMessage <<std::setprecision(3)<< L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< robust <<std::endl;
       std::cout<<GridLogMessage <<fmt << std::endl;
-      std::cout<<GridLogMessage ;
+      std::cout<<GridLogMessage;
 
       for(int i=0;i<mflops_all.size();i++){
 	std::cout<<mflops_all[i]/NN<<" ; " ;
@@ -456,7 +458,7 @@ public:
     return mflops_best;
   }
 
-  static double DWF(int Ls,int L)
+  static double DWF(int Ls,int L, double & robust)
   {
     RealD mass=0.1;
     RealD M5  =1.8;
@@ -658,10 +660,11 @@ public:
 	assert((norm2(err)<1.0e-4));
 
       }
+      robust = mflops_worst/mflops_best;
       std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
       std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Best  mflop/s        =   "<< mflops_best << " ; " << mflops_best/NN<<" per node " <<std::endl;
       std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Deo Worst mflop/s        =   "<< mflops_worst<< " ; " << mflops_worst/NN<<" per node " <<std::endl;
-      std::cout<<GridLogMessage << L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< mflops_worst/mflops_best <<std::endl;
+      std::cout<<GridLogMessage << std::fixed<<std::setprecision(3)<< L<<"^4 x "<<Ls<< " Performance Robustness   =   "<< robust  <<std::endl;
       std::cout<<GridLogMessage <<fmt << std::endl;
       std::cout<<GridLogMessage ;
 
@@ -701,7 +704,11 @@ int main (int argc, char ** argv)
 
   int sel=2;
   std::vector<int> L_list({8,12,16,24});
+
+  //int sel=1;
   //  std::vector<int> L_list({8,12});
+  std::vector<double> robust_list;
+
   std::vector<double> wilson;
   std::vector<double> dwf4;
   std::vector<double> dwf5;
@@ -712,7 +719,8 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << " Wilson dslash 4D vectorised" <<std::endl;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
     for(int l=0;l<L_list.size();l++){
-      wilson.push_back(Benchmark::DWF(1,L_list[l]));
+      double robust;
+      wilson.push_back(Benchmark::DWF(1,L_list[l],robust));
     }
   }
 
@@ -722,7 +730,10 @@ int main (int argc, char ** argv)
     std::cout<<GridLogMessage << " Domain wall dslash 4D vectorised" <<std::endl;
     std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
     for(int l=0;l<L_list.size();l++){
-      dwf4.push_back(Benchmark::DWF(Ls,L_list[l]));
+      double robust;
+      double result = Benchmark::DWF(Ls,L_list[l],robust) ;
+      dwf4.push_back(result);
+      robust_list.push_back(robust);
     }
   }
 
@@ -763,7 +774,6 @@ int main (int argc, char ** argv)
     Benchmark::Comms();
   }
 
-
   if ( do_dwf ) {
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
   std::cout<<GridLogMessage << " Per Node Summary table Ls="<<Ls <<std::endl;
@@ -775,7 +785,9 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << " Comparison point result: "  << dwf4[sel]/NN <<std::endl;
+  std::cout<<GridLogMessage << " Comparison point     result: "  << dwf4[sel]/NN <<std::endl;
+  std::cout<<std::setprecision(3);
+  std::cout<<GridLogMessage << " Comparison point robustness: "  << robust_list[sel]/NN <<std::endl;
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
   }
diff --git a/lib/allocator/AlignedAllocator.h b/lib/allocator/AlignedAllocator.h
index c5ad0883..f5bcb7a1 100644
--- a/lib/allocator/AlignedAllocator.h
+++ b/lib/allocator/AlignedAllocator.h
@@ -204,10 +204,12 @@ public:
 #endif
     size_type bytes = __n*sizeof(_Tp);
     uint8_t *cp = (uint8_t *)ptr;
+    if ( ptr ) { 
     // One touch per 4k page, static OMP loop to catch same loop order
 #pragma omp parallel for schedule(static)
-    for(size_type n=0;n<bytes;n+=4096){
-      cp[n]=0;
+      for(size_type n=0;n<bytes;n+=4096){
+	cp[n]=0;
+      }
     }
     return ptr;
   }
diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 956de0d2..50f0673f 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -37,7 +37,7 @@ namespace Grid {
 // Info that is setup once and indept of cartesian layout
 ///////////////////////////////////////////////////////////////
 void *              CartesianCommunicator::ShmCommBuf;
-uint64_t            CartesianCommunicator::MAX_MPI_SHM_BYTES   = 128*1024*1024; 
+uint64_t            CartesianCommunicator::MAX_MPI_SHM_BYTES   = 1024LL*1024LL*1024LL; 
 CartesianCommunicator::CommunicatorPolicy_t  
 CartesianCommunicator::CommunicatorPolicy= CartesianCommunicator::CommunicatorPolicyConcurrent;
 int CartesianCommunicator::nCommThreads = -1;
diff --git a/lib/util/Init.cc b/lib/util/Init.cc
index 3fd8b4cd..3232d32f 100644
--- a/lib/util/Init.cc
+++ b/lib/util/Init.cc
@@ -219,7 +219,8 @@ void Grid_init(int *argc,char ***argv)
     int MB;
     arg= GridCmdOptionPayload(*argv,*argv+*argc,"--shm");
     GridCmdOptionInt(arg,MB);
-    CartesianCommunicator::MAX_MPI_SHM_BYTES = MB*1024*1024;
+    uint64_t MB64 = MB;
+    CartesianCommunicator::MAX_MPI_SHM_BYTES = MB64*1024LL*1024LL;
   }
 
   if( GridCmdOptionExists(*argv,*argv+*argc,"--shm-hugepages") ){

From 49c20a9fa8e9afcd8d05aacd4154baee198605d5 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 31 Aug 2017 11:32:21 +0100
Subject: [PATCH 084/115] Patch to reporting

---
 benchmarks/Benchmark_ITT.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 3b064e19..ff2390ef 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -787,7 +787,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
   std::cout<<GridLogMessage << " Comparison point     result: "  << dwf4[sel]/NN <<std::endl;
   std::cout<<std::setprecision(3);
-  std::cout<<GridLogMessage << " Comparison point robustness: "  << robust_list[sel]/NN <<std::endl;
+  std::cout<<GridLogMessage << " Comparison point robustness: "  << robust_list[sel] <<std::endl;
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
   }

From b331be91018c78bcdab960cb369d841d8452df28 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Thu, 31 Aug 2017 11:32:57 +0100
Subject: [PATCH 085/115] Better reporting

---
 benchmarks/Benchmark_ITT.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index ff2390ef..2017f6d7 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -785,7 +785,7 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << " Comparison point     result: "  << dwf4[sel]/NN <<std::endl;
+  std::cout<<GridLogMessage << " Comparison point     result: "  << dwf4[sel]/NN << " Mflop/s per node"<<std::endl;
   std::cout<<std::setprecision(3);
   std::cout<<GridLogMessage << " Comparison point robustness: "  << robust_list[sel] <<std::endl;
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;

From b83b2b1415e8e4276486bff22230c56b9866e419 Mon Sep 17 00:00:00 2001
From: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Mon, 4 Sep 2017 14:09:47 +0100
Subject: [PATCH 086/115] Stability improvement to BCG. Force m_rr hermitian
 beyond rounding.

---
 .../iterative/BlockConjugateGradient.h        | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/lib/algorithms/iterative/BlockConjugateGradient.h b/lib/algorithms/iterative/BlockConjugateGradient.h
index d7817c05..e0eeddcb 100644
--- a/lib/algorithms/iterative/BlockConjugateGradient.h
+++ b/lib/algorithms/iterative/BlockConjugateGradient.h
@@ -87,15 +87,22 @@ void ThinQRfact (Eigen::MatrixXcd &m_rr,
   ////////////////////////////////////////////////////////////////////////////////////////////////////
   sliceInnerProductMatrix(m_rr,R,R,Orthog);
 
-  ////////////////////////////////////////////////////////////////////////////////////////////////////
-  // Cholesky from Eigen
-  // There exists a ldlt that is documented as more stable
-  ////////////////////////////////////////////////////////////////////////////////////////////////////
-  Eigen::MatrixXcd L    = m_rr.llt().matrixL(); 
+  // Force manifest hermitian to avoid rounding related
+  m_rr = 0.5*(m_rr+m_rr.adjoint());
 
+#if 0
+  std::cout << " Calling Cholesky  ldlt on m_rr "  << m_rr <<std::endl;
+  Eigen::MatrixXcd L_ldlt = m_rr.ldlt().matrixL(); 
+  std::cout << " Called Cholesky  ldlt on m_rr "  << L_ldlt <<std::endl;
+  auto  D_ldlt = m_rr.ldlt().vectorD(); 
+  std::cout << " Called Cholesky  ldlt on m_rr "  << D_ldlt <<std::endl;
+#endif
+
+  //  std::cout << " Calling Cholesky  llt on m_rr "  <<std::endl;
+  Eigen::MatrixXcd L    = m_rr.llt().matrixL(); 
+  //  std::cout << " Called Cholesky  llt on m_rr "  << L <<std::endl;
   C    = L.adjoint();
   Cinv = C.inverse();
-
   ////////////////////////////////////////////////////////////////////////////////////////////////////
   // Q = R C^{-1}
   //
@@ -103,7 +110,6 @@ void ThinQRfact (Eigen::MatrixXcd &m_rr,
   //
   // NB maddMatrix conventions are Right multiplication X[j] a[j,i] already
   ////////////////////////////////////////////////////////////////////////////////////////////////////
-  // FIXME:: make a sliceMulMatrix to avoid zero vector
   sliceMulMatrix(Q,Cinv,R,Orthog);
 }
 ////////////////////////////////////////////////////////////////////////////////////////////////////

From c05b2199f6f3d10ceedda091b9527a78ab1cef05 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Mon, 4 Sep 2017 10:41:21 -0400
Subject: [PATCH 087/115] Improvements to huge memory

---
 lib/communicator/Communicator_base.cc |  1 +
 lib/communicator/Communicator_mpi3.cc | 16 ++++++++++------
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 50f0673f..600b890b 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -157,6 +157,7 @@ void CartesianCommunicator::ShmInitGeneric(void){
     perror("mmap failed ");
     exit(EXIT_FAILURE);  
   }
+  if (!Hugepages ) madvise(ShmCommBuf,MAX_MPI_SHM_BYTES,MADV_HUGEPAGE);
 #else 
   ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES);
   ShmCommBuf=(void *)&ShmBufStorageVector[0];
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index cb7fa390..77042be0 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -198,7 +198,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   ShmCommBuf = 0;
   ShmCommBufs.resize(ShmSize);
 
-#if 1
+#if 0
   char shm_name [NAME_MAX];
   if ( ShmRank == 0 ) {
     for(int r=0;r<ShmSize;r++){
@@ -240,7 +240,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
 	  if (ierr && (page==0)) perror("numa relocate command failed");
 	}
 #endif
-      ShmCommBufs[r] =ptr;
+	ShmCommBufs[r] =ptr;
       
     }
   }
@@ -269,18 +269,22 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   if ( ShmRank == 0 ) {
     for(int r=0;r<ShmSize;r++){
       size_t size = CartesianCommunicator::MAX_MPI_SHM_BYTES;
-      key_t key   = 0x4545 + r;
+      key_t key   = IPC_PRIVATE;
       int flags = IPC_CREAT | SHM_R | SHM_W;
 #ifdef SHM_HUGETLB
-      flags|=SHM_HUGETLB;
+      if (Hugepages) flags|=SHM_HUGETLB;
 #endif
-      if ((shmids[r]= shmget(key,size, flags)) < 0) {
+      if ((shmids[r]= shmget(key,size, flags)) ==-1) {
 	int errsv = errno;
 	printf("Errno %d\n",errsv);
+	printf("key   %d\n",key);
+	printf("size  %lld\n",size);
+	printf("flags %d\n",flags);
 	perror("shmget");
 	exit(1);
+      } else { 
+	printf("shmid: 0x%x\n", shmids[r]);
       }
-      printf("shmid: 0x%x\n", shmids[r]);
     }
   }
   MPI_Barrier(ShmComm);

From 837bf8a5be1ed281e336412e865a94ceb62c5ee5 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Tue, 5 Sep 2017 12:51:02 +0100
Subject: [PATCH 088/115] Updating to control the SHM allocation scheme under
 configure time options

---
 configure.ac                          | 35 ++++++++++++++++++++++
 lib/communicator/Communicator_mpi3.cc | 43 ++++++++++++++++++++++++---
 2 files changed, 74 insertions(+), 4 deletions(-)

diff --git a/configure.ac b/configure.ac
index 134db759..b11d6b42 100644
--- a/configure.ac
+++ b/configure.ac
@@ -331,8 +331,41 @@ case ${ac_PRECISION} in
      double)
        AC_DEFINE([GRID_DEFAULT_PRECISION_DOUBLE],[1],[GRID_DEFAULT_PRECISION is DOUBLE] )
      ;;
+     *)
+     AC_MSG_ERROR([${ac_PRECISION} unsupported --enable-precision option]);
+     ;;
 esac
 
+######################  Shared memory allocation technique under MPI3
+AC_ARG_ENABLE([shm],[AC_HELP_STRING([--enable-shm=shmget|shmopen|hugetlbfs],
+              [Select SHM allocation technique])],[ac_SHM=${enable_shm}],[ac_SHM=shmopen])
+
+case ${ac_SHM} in
+
+     shmget)
+     AC_DEFINE([GRID_MPI3_SHMGET],[1],[GRID_MPI3_SHMGET] )
+     ;;
+
+     shmopen)
+     AC_DEFINE([GRID_MPI3_SHMOPEN],[1],[GRID_MPI3_SHMOPEN] )
+     ;;
+
+     hugetlbfs)
+     AC_DEFINE([GRID_MPI3_SHMMMAP],[1],[GRID_MPI3_SHMMMAP] )
+     ;;
+
+     *)
+     AC_MSG_ERROR([${ac_SHM} unsupported --enable-shm option]);
+     ;;
+esac
+
+######################  Shared base path for SHMMMAP
+AC_ARG_ENABLE([shmpath],[AC_HELP_STRING([--enable-shmpath=path],
+              [Select SHM mmap base path for hugetlbfs])],
+	      [ac_SHMPATH=${enable_shmpath}],
+	      [ac_SHMPATH=/var/lib/hugetlbfs/pagesize-2MB/])
+AC_DEFINE_UNQUOTED([GRID_SHM_PATH],["$ac_SHMPATH"],[Path to a hugetlbfs filesystem for MMAPing])
+
 ############### communication type selection
 AC_ARG_ENABLE([comms],[AC_HELP_STRING([--enable-comms=none|mpi|mpi-auto|mpi3|mpi3-auto|shmem],
               [Select communications])],[ac_COMMS=${enable_comms}],[ac_COMMS=none])
@@ -482,6 +515,8 @@ compiler version            : ${ax_cv_gxx_version}
 SIMD                        : ${ac_SIMD}${SIMD_GEN_WIDTH_MSG}
 Threading                   : ${ac_openmp}
 Communications type         : ${comms_type}
+Shared memory allocator     : ${ac_SHM}
+Shared memory mmap path     : ${ac_SHMPATH}
 Default precision           : ${ac_PRECISION}
 Software FP16 conversion    : ${ac_SFW_FP16}
 RNG choice                  : ${ac_RNG}
diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 77042be0..abcbec36 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -198,7 +198,39 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   ShmCommBuf = 0;
   ShmCommBufs.resize(ShmSize);
 
-#if 0
+  ////////////////////////////////////////////////////////////////////////////////////////////
+  // Hugetlbf and others map filesystems as mappable huge pages
+  ////////////////////////////////////////////////////////////////////////////////////////////
+#ifdef GRID_MPI3_SHMMMAP
+  char shm_name [NAME_MAX];
+  for(int r=0;r<ShmSize;r++){
+    
+    size_t size = CartesianCommunicator::MAX_MPI_SHM_BYTES;
+    sprintf(shm_name,GRID_SHM_PATH "/Grid_mpi3_shm_%d_%d",GroupRank,r);
+    int fd=open(shm_name,O_RDWR|O_CREAT,0666);
+    if ( fd == -1) { 
+      printf("open %s failed\n",shm_name);
+      perror("open hugetlbfs");
+      exit(0);
+    }
+    
+    int mmap_flag = MAP_SHARED ;
+#ifdef MAP_HUGETLB
+    if ( Hugepages ) mmap_flag |= MAP_HUGETLB;
+#endif
+    void *ptr = (void *) mmap(NULL, MAX_MPI_SHM_BYTES, PROT_READ | PROT_WRITE, mmap_flag,fd, 0); 
+    if ( ptr == (void *)MAP_FAILED ) {       perror("failed mmap");      assert(0);    }
+    assert(((uint64_t)ptr&0x3F)==0);
+    ShmCommBufs[r] =ptr;
+    
+  }
+#endif
+  ////////////////////////////////////////////////////////////////////////////////////////////
+  // POSIX SHMOPEN ; as far as I know Linux does not allow EXPLICIT HugePages with this case
+  // tmpfs (Larry Meadows says) does not support explicit huge page, and this is used for 
+  // the posix shm virtual file system
+  ////////////////////////////////////////////////////////////////////////////////////////////
+#ifdef GRID_MPI3_SHMOPEN
   char shm_name [NAME_MAX];
   if ( ShmRank == 0 ) {
     for(int r=0;r<ShmSize;r++){
@@ -218,7 +250,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
 #endif
       void * ptr =  mmap(NULL,size, PROT_READ | PROT_WRITE, mmap_flag, fd, 0);
 
-      if ( ptr == MAP_FAILED ) {       perror("failed mmap");      assert(0);    }
+      if ( ptr == (void * )MAP_FAILED ) {       perror("failed mmap");      assert(0);    }
       assert(((uint64_t)ptr&0x3F)==0);
 
 // Experiments; Experiments; Try to force numa domain on the shm segment if we have numaif.h
@@ -262,8 +294,11 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
       ShmCommBufs[r] =ptr;
     }
   }
-
-#else
+#endif
+  ////////////////////////////////////////////////////////////////////////////////////////////
+  // SHMGET SHMAT and SHM_HUGETLB flag
+  ////////////////////////////////////////////////////////////////////////////////////////////
+#ifdef GRID_MPI3_SHMGET
   std::vector<int> shmids(ShmSize);
 
   if ( ShmRank == 0 ) {

From bf973d0d56b0d4a945e8deab55d7310991de0861 Mon Sep 17 00:00:00 2001
From: Peter Boyle <paboyle@ph.ed.ac.uk>
Date: Tue, 5 Sep 2017 14:30:29 +0100
Subject: [PATCH 089/115] SHM complete

---
 lib/communicator/Communicator_mpi3.cc | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index abcbec36..75d1f000 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -207,6 +207,8 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
     
     size_t size = CartesianCommunicator::MAX_MPI_SHM_BYTES;
     sprintf(shm_name,GRID_SHM_PATH "/Grid_mpi3_shm_%d_%d",GroupRank,r);
+    //sprintf(shm_name,"/var/lib/hugetlbfs/group/wheel/pagesize-2MB/" "Grid_mpi3_shm_%d_%d",GroupRank,r);
+    //    printf("Opening file %s \n",shm_name);
     int fd=open(shm_name,O_RDWR|O_CREAT,0666);
     if ( fd == -1) { 
       printf("open %s failed\n",shm_name);
@@ -214,12 +216,15 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
       exit(0);
     }
     
-    int mmap_flag = MAP_SHARED ;
+    int mmap_flag = MAP_SHARED |MAP_POPULATE;
 #ifdef MAP_HUGETLB
     if ( Hugepages ) mmap_flag |= MAP_HUGETLB;
 #endif
     void *ptr = (void *) mmap(NULL, MAX_MPI_SHM_BYTES, PROT_READ | PROT_WRITE, mmap_flag,fd, 0); 
-    if ( ptr == (void *)MAP_FAILED ) {       perror("failed mmap");      assert(0);    }
+    if ( ptr == (void *)MAP_FAILED ) {    
+      printf("mmap %s failed\n",shm_name);
+      perror("failed mmap");      assert(0);    
+    }
     assert(((uint64_t)ptr&0x3F)==0);
     ShmCommBufs[r] =ptr;
     
@@ -244,7 +249,7 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
       if ( fd < 0 ) {	perror("failed shm_open");	assert(0);      }
       ftruncate(fd, size);
       
-      int mmap_flag = MAP_SHARED;
+      int mmap_flag = MAP_SHARED|MAP_POPULATE;
 #ifdef MAP_HUGETLB
       if (Hugepages) mmap_flag |= MAP_HUGETLB;
 #endif

From ba27cc657183e46a08444f0fe961b9af3a77a8f3 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 5 Sep 2017 15:00:16 +0100
Subject: [PATCH 090/115] Mac os happiness

---
 lib/communicator/Communicator_base.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 600b890b..20c310c0 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -157,7 +157,9 @@ void CartesianCommunicator::ShmInitGeneric(void){
     perror("mmap failed ");
     exit(EXIT_FAILURE);  
   }
+#ifdef MADV_HUGEPAGE
   if (!Hugepages ) madvise(ShmCommBuf,MAX_MPI_SHM_BYTES,MADV_HUGEPAGE);
+#endif
 #else 
   ShmBufStorageVector.resize(MAX_MPI_SHM_BYTES);
   ShmCommBuf=(void *)&ShmBufStorageVector[0];

From aed5de4d508c990b2d22edbf2fb64526c8095552 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 5 Sep 2017 15:07:07 +0100
Subject: [PATCH 091/115] Patching macos compile

---
 lib/communicator/Communicator_mpi3.cc | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/communicator/Communicator_mpi3.cc b/lib/communicator/Communicator_mpi3.cc
index 75d1f000..44aa1024 100644
--- a/lib/communicator/Communicator_mpi3.cc
+++ b/lib/communicator/Communicator_mpi3.cc
@@ -215,8 +215,10 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
       perror("open hugetlbfs");
       exit(0);
     }
-    
-    int mmap_flag = MAP_SHARED |MAP_POPULATE;
+    int mmap_flag = MAP_SHARED ;
+#ifdef MAP_POPULATE    
+    mmap_flag|=MAP_POPULATE;
+#endif
 #ifdef MAP_HUGETLB
     if ( Hugepages ) mmap_flag |= MAP_HUGETLB;
 #endif
@@ -249,7 +251,10 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
       if ( fd < 0 ) {	perror("failed shm_open");	assert(0);      }
       ftruncate(fd, size);
       
-      int mmap_flag = MAP_SHARED|MAP_POPULATE;
+      int mmap_flag = MAP_SHARED;
+#ifdef MAP_POPULATE 
+      mmap_flag |= MAP_POPULATE;
+#endif
 #ifdef MAP_HUGETLB
       if (Hugepages) mmap_flag |= MAP_HUGETLB;
 #endif

From 7cb2b11f269a12b6f0874708be00fb4237258c1e Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Fri, 8 Sep 2017 13:41:53 +0100
Subject: [PATCH 092/115] Fixing Intel compiler error for the JSON parser

---
 lib/serialisation/JSON_IO.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/serialisation/JSON_IO.h b/lib/serialisation/JSON_IO.h
index c82648fc..23b9a836 100644
--- a/lib/serialisation/JSON_IO.h
+++ b/lib/serialisation/JSON_IO.h
@@ -120,7 +120,7 @@ namespace Grid
      ss_ << os.str() << " ," ;
   }
 
-/*
+
   // specialize for string
   template <>
   void JSONWriter::writeDefault(const std::string &s, const std::string &x)
@@ -133,7 +133,7 @@ namespace Grid
     else
      ss_ << os.str() << " ," ;
   }
-*/
+
 
 
   template <typename U>

From 1950ac92940b85aa32dbd4ce066e68f4e7220d45 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Fri, 8 Sep 2017 15:18:59 +0100
Subject: [PATCH 093/115] Fixed the Intel compiler problem with the JSON
 classes

---
 lib/json/json.hpp            | 14281 +++++++++++++++++----------------
 lib/serialisation/JSON_IO.cc |     3 +-
 lib/serialisation/JSON_IO.h  |    18 +-
 3 files changed, 7226 insertions(+), 7076 deletions(-)

diff --git a/lib/json/json.hpp b/lib/json/json.hpp
index e53fbcf2..e7c42920 100644
--- a/lib/json/json.hpp
+++ b/lib/json/json.hpp
@@ -29,7 +29,7 @@ SOFTWARE.
 #ifndef NLOHMANN_JSON_HPP
 #define NLOHMANN_JSON_HPP
 
-#include <algorithm> // all_of, copy, fill, find, for_each, none_of, remove, reverse, transform
+#include <algorithm> // all_of, copy, fill, find, for_each, generate_n, none_of, remove, reverse, transform
 #include <array> // array
 #include <cassert> // assert
 #include <ciso646> // and, not, or
@@ -43,7 +43,7 @@ SOFTWARE.
 #include <functional> // function, hash, less
 #include <initializer_list> // initializer_list
 #include <iomanip> // hex
-#include <iostream> // istream, ostream
+#include <iosfwd>   // istream, ostream
 #include <iterator> // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
 #include <limits> // numeric_limits
 #include <locale> // locale
@@ -54,6 +54,7 @@ SOFTWARE.
 #include <string> // getline, stoi, string, to_string
 #include <type_traits> // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type
 #include <utility> // declval, forward, make_pair, move, pair, swap
+#include <valarray> // valarray
 #include <vector> // vector
 
 // exclude unsupported compilers
@@ -115,6 +116,38 @@ SOFTWARE.
 */
 namespace nlohmann
 {
+template<typename = void, typename = void>
+struct adl_serializer;
+
+// forward declaration of basic_json (required to split the class)
+template<template<typename U, typename V, typename... Args> class ObjectType =
+         std::map,
+         template<typename U, typename... Args> class ArrayType = std::vector,
+         class StringType = std::string, class BooleanType = bool,
+         class NumberIntegerType = std::int64_t,
+         class NumberUnsignedType = std::uint64_t,
+         class NumberFloatType = double,
+         template<typename U> class AllocatorType = std::allocator,
+         template<typename T, typename SFINAE = void> class JSONSerializer =
+         adl_serializer>
+class basic_json;
+
+// Ugly macros to avoid uglier copy-paste when specializing basic_json
+// This is only temporary and will be removed in 3.0
+
+#define NLOHMANN_BASIC_JSON_TPL_DECLARATION                                \
+    template<template<typename, typename, typename...> class ObjectType,   \
+             template<typename, typename...> class ArrayType,              \
+             class StringType, class BooleanType, class NumberIntegerType, \
+             class NumberUnsignedType, class NumberFloatType,              \
+             template<typename> class AllocatorType,                       \
+             template<typename, typename = void> class JSONSerializer>
+
+#define NLOHMANN_BASIC_JSON_TPL                                            \
+    basic_json<ObjectType, ArrayType, StringType, BooleanType,             \
+    NumberIntegerType, NumberUnsignedType, NumberFloatType,                \
+    AllocatorType, JSONSerializer>
+
 
 /*!
 @brief unnamed namespace with internal helper functions
@@ -133,12 +166,28 @@ namespace detail
 /*!
 @brief general exception of the @ref basic_json class
 
-Extension of std::exception objects with a member @a id for exception ids.
+This class is an extension of `std::exception` objects with a member @a id for
+exception ids. It is used as the base class for all exceptions thrown by the
+@ref basic_json class. This class can hence be used as "wildcard" to catch
+exceptions.
 
+Subclasses:
+- @ref parse_error for exceptions indicating a parse error
+- @ref invalid_iterator for exceptions indicating errors with iterators
+- @ref type_error for exceptions indicating executing a member function with
+                  a wrong type
+- @ref out_of_range for exceptions indicating access out of the defined range
+- @ref other_error for exceptions indicating other library errors
+
+@internal
 @note To have nothrow-copy-constructible exceptions, we internally use
-      std::runtime_error which can cope with arbitrary-length error messages.
+      `std::runtime_error` which can cope with arbitrary-length error messages.
       Intermediate strings are built with static functions and then passed to
       the actual constructor.
+@endinternal
+
+@liveexample{The following code shows how arbitrary library exceptions can be
+caught.,exception}
 
 @since version 3.0.0
 */
@@ -146,7 +195,7 @@ class exception : public std::exception
 {
   public:
     /// returns the explanatory string
-    virtual const char* what() const noexcept override
+    const char* what() const noexcept override
     {
         return m.what();
     }
@@ -155,9 +204,7 @@ class exception : public std::exception
     const int id;
 
   protected:
-    exception(int id_, const char* what_arg)
-        : id(id_), m(what_arg)
-    {}
+    exception(int id_, const char* what_arg) : id(id_), m(what_arg) {}
 
     static std::string name(const std::string& ename, int id)
     {
@@ -172,21 +219,16 @@ class exception : public std::exception
 /*!
 @brief exception indicating a parse error
 
-This excpetion is thrown by the library when a parse error occurs. Parse
-errors can occur during the deserialization of JSON text as well as when
-using JSON Patch.
+This excpetion is thrown by the library when a parse error occurs. Parse errors
+can occur during the deserialization of JSON text, CBOR, MessagePack, as well
+as when using JSON Patch.
 
 Member @a byte holds the byte index of the last read character in the input
 file.
 
-@note For an input with n bytes, 1 is the index of the first character
-      and n+1 is the index of the terminating null byte or the end of
-      file. This also holds true when reading a byte vector (CBOR or
-      MessagePack).
-
 Exceptions have ids 1xx.
 
-name / id                      | example massage | description
+name / id                      | example message | description
 ------------------------------ | --------------- | -------------------------
 json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position.
 json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point.
@@ -198,10 +240,23 @@ json.exception.parse_error.107 | parse error: JSON pointer must be empty or begi
 json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences.
 json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number.
 json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read.
-json.exception.parse_error.111 | parse error: bad input stream | Parsing CBOR or MessagePack from an input stream where the [`badbit` or `failbit`](http://en.cppreference.com/w/cpp/io/ios_base/iostate) is set.
 json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xf8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read.
 json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read.
 
+@note For an input with n bytes, 1 is the index of the first character and n+1
+      is the index of the terminating null byte or the end of file. This also
+      holds true when reading a byte vector (CBOR or MessagePack).
+
+@liveexample{The following code shows how a `parse_error` exception can be
+caught.,parse_error}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
+
 @since version 3.0.0
 */
 class parse_error : public exception
@@ -209,13 +264,13 @@ class parse_error : public exception
   public:
     /*!
     @brief create a parse error exception
-    @param[in] id         the id of the exception
-    @param[in] byte_      the byte index where the error occured (or 0 if
-                          the position cannot be determined)
-    @param[in] what_arg   the explanatory string
+    @param[in] id        the id of the exception
+    @param[in] byte_     the byte index where the error occurred (or 0 if the
+                         position cannot be determined)
+    @param[in] what_arg  the explanatory string
     @return parse_error object
     */
-    static parse_error create(int id, size_t byte_, const std::string& what_arg)
+    static parse_error create(int id, std::size_t byte_, const std::string& what_arg)
     {
         std::string w = exception::name("parse_error", id) + "parse error" +
                         (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") +
@@ -228,25 +283,26 @@ class parse_error : public exception
 
     The byte index of the last read character in the input file.
 
-    @note For an input with n bytes, 1 is the index of the first character
-          and n+1 is the index of the terminating null byte or the end of
-          file. This also holds true when reading a byte vector (CBOR or
-          MessagePack).
+    @note For an input with n bytes, 1 is the index of the first character and
+          n+1 is the index of the terminating null byte or the end of file.
+          This also holds true when reading a byte vector (CBOR or MessagePack).
     */
-    const size_t byte;
+    const std::size_t byte;
 
   private:
-    parse_error(int id_, size_t byte_, const char* what_arg)
-        : exception(id_, what_arg), byte(byte_)
-    {}
+    parse_error(int id_, std::size_t byte_, const char* what_arg)
+        : exception(id_, what_arg), byte(byte_) {}
 };
 
 /*!
 @brief exception indicating errors with iterators
 
+This exception is thrown if iterators passed to a library function do not match
+the expected semantics.
+
 Exceptions have ids 2xx.
 
-name / id                           | example massage | description
+name / id                           | example message | description
 ----------------------------------- | --------------- | -------------------------
 json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
 json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion.
@@ -260,9 +316,19 @@ json.exception.invalid_iterator.209 | cannot use offsets with object iterators |
 json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid.
 json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to.
 json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container.
-json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compated, because JSON objects are unordered.
+json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered.
 json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin().
 
+@liveexample{The following code shows how an `invalid_iterator` exception can be
+caught.,invalid_iterator}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
+
 @since version 3.0.0
 */
 class invalid_iterator : public exception
@@ -276,13 +342,15 @@ class invalid_iterator : public exception
 
   private:
     invalid_iterator(int id_, const char* what_arg)
-        : exception(id_, what_arg)
-    {}
+        : exception(id_, what_arg) {}
 };
 
 /*!
 @brief exception indicating executing a member function with a wrong type
 
+This exception is thrown in case of a type error; that is, a library function is
+executed on a JSON value whose type does not match the expected semantics.
+
 Exceptions have ids 3xx.
 
 name / id                     | example message | description
@@ -298,10 +366,20 @@ json.exception.type_error.308 | cannot use push_back() with string | The @ref pu
 json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types.
 json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types.
 json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types.
+json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types.
 json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined.
 json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers.
 json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive.
 
+@liveexample{The following code shows how a `type_error` exception can be
+caught.,type_error}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+@sa @ref other_error for exceptions indicating other library errors
+
 @since version 3.0.0
 */
 class type_error : public exception
@@ -314,14 +392,16 @@ class type_error : public exception
     }
 
   private:
-    type_error(int id_, const char* what_arg)
-        : exception(id_, what_arg)
-    {}
+    type_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
 };
 
 /*!
 @brief exception indicating access out of the defined range
 
+This exception is thrown in case a library function is called on an input
+parameter that exceeds the expected range, for instance in case of array
+indices or nonexisting object keys.
+
 Exceptions have ids 4xx.
 
 name / id                       | example message | description
@@ -333,6 +413,16 @@ json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference
 json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value.
 json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF.
 
+@liveexample{The following code shows how an `out_of_range` exception can be
+caught.,out_of_range}
+
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref other_error for exceptions indicating other library errors
+
 @since version 3.0.0
 */
 class out_of_range : public exception
@@ -345,13 +435,14 @@ class out_of_range : public exception
     }
 
   private:
-    out_of_range(int id_, const char* what_arg)
-        : exception(id_, what_arg)
-    {}
+    out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {}
 };
 
 /*!
-@brief exception indicating other errors
+@brief exception indicating other library errors
+
+This exception is thrown in case of errors that cannot be classified with the
+other exception types.
 
 Exceptions have ids 5xx.
 
@@ -360,6 +451,16 @@ name / id                      | example message | description
 json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed.
 json.exception.other_error.502 | invalid object size for conversion | Some conversions to user-defined types impose constraints on the object size (e.g. std::pair)
 
+@sa @ref exception for the base class of the library exceptions
+@sa @ref parse_error for exceptions indicating a parse error
+@sa @ref invalid_iterator for exceptions indicating errors with iterators
+@sa @ref type_error for exceptions indicating executing a member function with
+                    a wrong type
+@sa @ref out_of_range for exceptions indicating access out of the defined range
+
+@liveexample{The following code shows how an `other_error` exception can be
+caught.,other_error}
+
 @since version 3.0.0
 */
 class other_error : public exception
@@ -372,9 +473,7 @@ class other_error : public exception
     }
 
   private:
-    other_error(int id_, const char* what_arg)
-        : exception(id_, what_arg)
-    {}
+    other_error(int id_, const char* what_arg) : exception(id_, what_arg) {}
 };
 
 
@@ -409,15 +508,15 @@ value with the default value for a given type
 */
 enum class value_t : uint8_t
 {
-    null,            ///< null value
-    object,          ///< object (unordered set of name/value pairs)
-    array,           ///< array (ordered collection of values)
-    string,          ///< string value
-    boolean,         ///< boolean value
-    number_integer,  ///< number value (signed integer)
-    number_unsigned, ///< number value (unsigned integer)
-    number_float,    ///< number value (floating-point)
-    discarded        ///< discarded by the the parser callback function
+    null,             ///< null value
+    object,           ///< object (unordered set of name/value pairs)
+    array,            ///< array (ordered collection of values)
+    string,           ///< string value
+    boolean,          ///< boolean value
+    number_integer,   ///< number value (signed integer)
+    number_unsigned,  ///< number value (unsigned integer)
+    number_float,     ///< number value (floating-point)
+    discarded         ///< discarded by the the parser callback function
 };
 
 /*!
@@ -444,13 +543,8 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept
     };
 
     // discarded values are not comparable
-    if (lhs == value_t::discarded or rhs == value_t::discarded)
-    {
-        return false;
-    }
-
-    return order[static_cast<std::size_t>(lhs)] <
-           order[static_cast<std::size_t>(rhs)];
+    return lhs != value_t::discarded and rhs != value_t::discarded and
+           order[static_cast<std::size_t>(lhs)] < order[static_cast<std::size_t>(rhs)];
 }
 
 
@@ -458,6 +552,11 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept
 // helpers //
 /////////////
 
+template<typename> struct is_basic_json : std::false_type {};
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {};
+
 // alias templates to reduce boilerplate
 template<bool B, typename T = void>
 using enable_if_t = typename std::enable_if<B, T>::type;
@@ -467,7 +566,7 @@ using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::typ
 
 // implementation of C++14 index_sequence and affiliates
 // source: https://stackoverflow.com/a/32223343
-template <std::size_t... Ints>
+template<std::size_t... Ints>
 struct index_sequence
 {
     using type = index_sequence;
@@ -478,19 +577,19 @@ struct index_sequence
     }
 };
 
-template <class Sequence1, class Sequence2>
+template<class Sequence1, class Sequence2>
 struct merge_and_renumber;
 
-template <std::size_t... I1, std::size_t... I2>
+template<std::size_t... I1, std::size_t... I2>
 struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>>
         : index_sequence < I1..., (sizeof...(I1) + I2)... >
-          { };
+          {};
 
-template <std::size_t N>
+template<std::size_t N>
 struct make_index_sequence
     : merge_and_renumber < typename make_index_sequence < N / 2 >::type,
       typename make_index_sequence < N - N / 2 >::type >
-{ };
+{};
 
 template<> struct make_index_sequence<0> : index_sequence<> { };
 template<> struct make_index_sequence<1> : index_sequence<0> { };
@@ -551,6 +650,14 @@ struct external_constructor<value_t::string>
         j.m_value = s;
         j.assert_invariant();
     }
+
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+    {
+        j.m_type = value_t::string;
+        j.m_value = std::move(s);
+        j.assert_invariant();
+    }
 };
 
 template<>
@@ -600,6 +707,14 @@ struct external_constructor<value_t::array>
         j.assert_invariant();
     }
 
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+    {
+        j.m_type = value_t::array;
+        j.m_value = std::move(arr);
+        j.assert_invariant();
+    }
+
     template<typename BasicJsonType, typename CompatibleArrayType,
              enable_if_t<not std::is_same<CompatibleArrayType,
                                           typename BasicJsonType::array_t>::value,
@@ -625,6 +740,18 @@ struct external_constructor<value_t::array>
         }
         j.assert_invariant();
     }
+
+    template<typename BasicJsonType, typename T,
+             enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+    static void construct(BasicJsonType& j, const std::valarray<T>& arr)
+    {
+        using std::begin;
+        using std::end;
+        j.m_type = value_t::array;
+        j.m_value = value_t::array;
+        j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr));
+        j.assert_invariant();
+    }
 };
 
 template<>
@@ -638,10 +765,17 @@ struct external_constructor<value_t::object>
         j.assert_invariant();
     }
 
+    template<typename BasicJsonType>
+    static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+    {
+        j.m_type = value_t::object;
+        j.m_value = std::move(obj);
+        j.assert_invariant();
+    }
+
     template<typename BasicJsonType, typename CompatibleObjectType,
              enable_if_t<not std::is_same<CompatibleObjectType,
-                                          typename BasicJsonType::object_t>::value,
-                         int> = 0>
+                                          typename BasicJsonType::object_t>::value, int> = 0>
     static void construct(BasicJsonType& j, const CompatibleObjectType& obj)
     {
         using std::begin;
@@ -694,10 +828,8 @@ template<class RealType, class CompatibleObjectType>
 struct is_compatible_object_type_impl<true, RealType, CompatibleObjectType>
 {
     static constexpr auto value =
-        std::is_constructible<typename RealType::key_type,
-        typename CompatibleObjectType::key_type>::value and
-        std::is_constructible<typename RealType::mapped_type,
-        typename CompatibleObjectType::mapped_type>::value;
+        std::is_constructible<typename RealType::key_type, typename CompatibleObjectType::key_type>::value and
+        std::is_constructible<typename RealType::mapped_type, typename CompatibleObjectType::mapped_type>::value;
 };
 
 template<class BasicJsonType, class CompatibleObjectType>
@@ -716,8 +848,7 @@ struct is_basic_json_nested_type
     static auto constexpr value = std::is_same<T, typename BasicJsonType::iterator>::value or
                                   std::is_same<T, typename BasicJsonType::const_iterator>::value or
                                   std::is_same<T, typename BasicJsonType::reverse_iterator>::value or
-                                  std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value or
-                                  std::is_same<T, typename BasicJsonType::json_pointer>::value;
+                                  std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value;
 };
 
 template<class BasicJsonType, class CompatibleArrayType>
@@ -745,8 +876,7 @@ struct is_compatible_integer_type_impl<true, RealIntegerType, CompatibleNumberIn
     using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>;
 
     static constexpr auto value =
-        std::is_constructible<RealIntegerType,
-        CompatibleNumberIntegerType>::value and
+        std::is_constructible<RealIntegerType, CompatibleNumberIntegerType>::value and
         CompatibleLimits::is_integer and
         RealLimits::is_signed == CompatibleLimits::is_signed;
 };
@@ -831,6 +961,12 @@ void to_json(BasicJsonType& j, const CompatibleString& s)
     external_constructor<value_t::string>::construct(j, s);
 }
 
+template <typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s)
+{
+    external_constructor<value_t::string>::construct(j, std::move(s));
+}
+
 template<typename BasicJsonType, typename FloatType,
          enable_if_t<std::is_floating_point<FloatType>::value, int> = 0>
 void to_json(BasicJsonType& j, FloatType val) noexcept
@@ -876,42 +1012,61 @@ template <
         is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or
         std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value,
         int > = 0 >
-void to_json(BasicJsonType& j, const  CompatibleArrayType& arr)
+void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
 {
     external_constructor<value_t::array>::construct(j, arr);
 }
 
+template <typename BasicJsonType, typename T,
+          enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
+void to_json(BasicJsonType& j, std::valarray<T> arr)
+{
+    external_constructor<value_t::array>::construct(j, std::move(arr));
+}
+
+template <typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr)
+{
+    external_constructor<value_t::array>::construct(j, std::move(arr));
+}
+
 template <
     typename BasicJsonType, typename CompatibleObjectType,
     enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value,
                 int> = 0 >
-void to_json(BasicJsonType& j, const  CompatibleObjectType& arr)
+void to_json(BasicJsonType& j, const CompatibleObjectType& obj)
 {
-    external_constructor<value_t::object>::construct(j, arr);
+    external_constructor<value_t::object>::construct(j, obj);
 }
 
-template <typename BasicJsonType, typename T, std::size_t N,
-          enable_if_t<not std::is_constructible<
-                          typename BasicJsonType::string_t, T (&)[N]>::value,
-                      int> = 0>
+template <typename BasicJsonType>
+void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj)
+{
+    external_constructor<value_t::object>::construct(j, std::move(obj));
+}
+
+template<typename BasicJsonType, typename T, std::size_t N,
+         enable_if_t<not std::is_constructible<
+                         typename BasicJsonType::string_t, T (&)[N]>::value,
+                     int> = 0>
 void to_json(BasicJsonType& j, T (&arr)[N])
 {
     external_constructor<value_t::array>::construct(j, arr);
 }
 
-template <typename BasicJsonType, typename... Args>
+template<typename BasicJsonType, typename... Args>
 void to_json(BasicJsonType& j, const std::pair<Args...>& p)
 {
     j = {p.first, p.second};
 }
 
-template <typename BasicJsonType, typename Tuple, std::size_t... Idx>
+template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
 void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...>)
 {
     j = {std::get<Idx>(t)...};
 }
 
-template <typename BasicJsonType, typename... Args>
+template<typename BasicJsonType, typename... Args>
 void to_json(BasicJsonType& j, const std::tuple<Args...>& t)
 {
     to_json_tuple_impl(j, t, index_sequence_for<Args...> {});
@@ -933,35 +1088,31 @@ void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val)
     {
         case value_t::number_unsigned:
         {
-            val = static_cast<ArithmeticType>(
-                      *j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>());
             break;
         }
         case value_t::number_integer:
         {
-            val = static_cast<ArithmeticType>(
-                      *j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>());
             break;
         }
         case value_t::number_float:
         {
-            val = static_cast<ArithmeticType>(
-                      *j.template get_ptr<const typename BasicJsonType::number_float_t*>());
+            val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>());
             break;
         }
+
         default:
-        {
-            JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name()));
-        }
+            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
     }
 }
 
 template<typename BasicJsonType>
 void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
 {
-    if (not j.is_boolean())
+    if (JSON_UNLIKELY(not j.is_boolean()))
     {
-        JSON_THROW(type_error::create(302, "type must be boolean, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name())));
     }
     b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>();
 }
@@ -969,9 +1120,9 @@ void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b)
 template<typename BasicJsonType>
 void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s)
 {
-    if (not j.is_string())
+    if (JSON_UNLIKELY(not j.is_string()))
     {
-        JSON_THROW(type_error::create(302, "type must be string, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name())));
     }
     s = *j.template get_ptr<const typename BasicJsonType::string_t*>();
 }
@@ -1006,9 +1157,9 @@ void from_json(const BasicJsonType& j, EnumType& e)
 template<typename BasicJsonType>
 void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr)
 {
-    if (not j.is_array())
+    if (JSON_UNLIKELY(not j.is_array()))
     {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
     }
     arr = *j.template get_ptr<const typename BasicJsonType::array_t*>();
 }
@@ -1018,21 +1169,36 @@ template<typename BasicJsonType, typename T, typename Allocator,
          enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
 void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l)
 {
-    if (not j.is_array())
+    if (JSON_UNLIKELY(not j.is_array()))
     {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
     }
-
-    for (auto it = j.rbegin(), end = j.rend(); it != end; ++it)
+    std::transform(j.rbegin(), j.rend(),
+                   std::front_inserter(l), [](const BasicJsonType & i)
     {
-        l.push_front(it->template get<T>());
+        return i.template get<T>();
+    });
+}
+
+// valarray doesn't have an insert method
+template<typename BasicJsonType, typename T,
+         enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0>
+void from_json(const BasicJsonType& j, std::valarray<T>& l)
+{
+    if (JSON_UNLIKELY(not j.is_array()))
+    {
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
+    }
+    l.resize(j.size());
+    for (size_t i = 0; i < j.size(); ++i)
+    {
+        l[i] = j[i];
     }
 }
 
 template<typename BasicJsonType, typename CompatibleArrayType>
-void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0>)
+void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0> /*unused*/)
 {
-    using std::begin;
     using std::end;
 
     std::transform(j.begin(), j.end(),
@@ -1045,12 +1211,11 @@ void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, prio
 }
 
 template<typename BasicJsonType, typename CompatibleArrayType>
-auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1>)
+auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1> /*unused*/)
 -> decltype(
     arr.reserve(std::declval<typename CompatibleArrayType::size_type>()),
     void())
 {
-    using std::begin;
     using std::end;
 
     arr.reserve(j.size());
@@ -1063,8 +1228,8 @@ auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, prio
     });
 }
 
-template <typename BasicJsonType, typename T, std::size_t N>
-void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2>)
+template<typename BasicJsonType, typename T, std::size_t N>
+void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2> /*unused*/)
 {
     for (std::size_t i = 0; i < N; ++i)
     {
@@ -1078,9 +1243,9 @@ template<typename BasicJsonType, typename CompatibleArrayType,
                      not std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, int> = 0>
 void from_json(const BasicJsonType& j, CompatibleArrayType& arr)
 {
-    if (not j.is_array())
+    if (JSON_UNLIKELY(not j.is_array()))
     {
-        JSON_THROW(type_error::create(302, "type must be array, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name())));
     }
 
     from_json_array_impl(j, arr, priority_tag<2> {});
@@ -1090,24 +1255,19 @@ template<typename BasicJsonType, typename CompatibleObjectType,
          enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0>
 void from_json(const BasicJsonType& j, CompatibleObjectType& obj)
 {
-    if (not j.is_object())
+    if (JSON_UNLIKELY(not j.is_object()))
     {
-        JSON_THROW(type_error::create(302, "type must be object, but is " + j.type_name()));
+        JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name())));
     }
 
     auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>();
-    using std::begin;
-    using std::end;
     using value_type = typename CompatibleObjectType::value_type;
     std::transform(
         inner_object->begin(), inner_object->end(),
         std::inserter(obj, obj.begin()),
         [](typename BasicJsonType::object_t::value_type const & p)
     {
-        return value_type(
-                   p.first,
-                   p.second
-                   .template get<typename CompatibleObjectType::mapped_type>());
+        return value_type(p.first, p.second.template get<typename CompatibleObjectType::mapped_type>());
     });
 }
 
@@ -1147,26 +1307,25 @@ void from_json(const BasicJsonType& j, ArithmeticType& val)
             val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>());
             break;
         }
+
         default:
-        {
-            JSON_THROW(type_error::create(302, "type must be number, but is " + j.type_name()));
-        }
+            JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name())));
     }
 }
 
-template <typename BasicJsonType, typename... Args>
-void from_json(const BasicJsonType& j, std::pair<Args...>& p)
+template<typename BasicJsonType, typename A1, typename A2>
+void from_json(const BasicJsonType& j, std::pair<A1, A2>& p)
 {
-    p = {j.at(0), j.at(1)};
+    p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()};
 }
 
-template <typename BasicJsonType, typename Tuple, std::size_t... Idx>
+template<typename BasicJsonType, typename Tuple, std::size_t... Idx>
 void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...>)
 {
-    t = std::make_tuple(j.at(Idx)...);
+    t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...);
 }
 
-template <typename BasicJsonType, typename... Args>
+template<typename BasicJsonType, typename... Args>
 void from_json(const BasicJsonType& j, std::tuple<Args...>& t)
 {
     from_json_tuple_impl(j, t, index_sequence_for<Args...> {});
@@ -1176,14 +1335,14 @@ struct to_json_fn
 {
   private:
     template<typename BasicJsonType, typename T>
-    auto call(BasicJsonType& j, T&& val, priority_tag<1>) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
+    auto call(BasicJsonType& j, T&& val, priority_tag<1> /*unused*/) const noexcept(noexcept(to_json(j, std::forward<T>(val))))
     -> decltype(to_json(j, std::forward<T>(val)), void())
     {
         return to_json(j, std::forward<T>(val));
     }
 
     template<typename BasicJsonType, typename T>
-    void call(BasicJsonType&, T&&, priority_tag<0>) const noexcept
+    void call(BasicJsonType& /*unused*/, T&& /*unused*/, priority_tag<0> /*unused*/) const noexcept
     {
         static_assert(sizeof(BasicJsonType) == 0,
                       "could not find to_json() method in T's namespace");
@@ -1202,7 +1361,7 @@ struct from_json_fn
 {
   private:
     template<typename BasicJsonType, typename T>
-    auto call(const BasicJsonType& j, T& val, priority_tag<1>) const
+    auto call(const BasicJsonType& j, T& val, priority_tag<1> /*unused*/) const
     noexcept(noexcept(from_json(j, val)))
     -> decltype(from_json(j, val), void())
     {
@@ -1210,7 +1369,7 @@ struct from_json_fn
     }
 
     template<typename BasicJsonType, typename T>
-    void call(const BasicJsonType&, T&, priority_tag<0>) const noexcept
+    void call(const BasicJsonType& /*unused*/, T& /*unused*/, priority_tag<0> /*unused*/) const noexcept
     {
         static_assert(sizeof(BasicJsonType) == 0,
                       "could not find from_json() method in T's namespace");
@@ -1234,8 +1393,5478 @@ struct static_const
 
 template<typename T>
 constexpr T static_const<T>::value;
-} // namespace detail
 
+////////////////////
+// input adapters //
+////////////////////
+
+/// abstract input adapter interface
+struct input_adapter_protocol
+{
+    virtual int get_character() = 0;
+    virtual std::string read(std::size_t offset, std::size_t length) = 0;
+    virtual ~input_adapter_protocol() = default;
+};
+
+/// a type to simplify interfaces
+using input_adapter_t = std::shared_ptr<input_adapter_protocol>;
+
+/// input adapter for cached stream input
+template<std::size_t BufferSize>
+class cached_input_stream_adapter : public input_adapter_protocol
+{
+  public:
+    explicit cached_input_stream_adapter(std::istream& i)
+        : is(i), start_position(is.tellg())
+    {
+        fill_buffer();
+
+        // skip byte order mark
+        if (fill_size >= 3 and buffer[0] == '\xEF' and buffer[1] == '\xBB' and buffer[2] == '\xBF')
+        {
+            buffer_pos += 3;
+            processed_chars += 3;
+        }
+    }
+
+    ~cached_input_stream_adapter() override
+    {
+        // clear stream flags
+        is.clear();
+        // We initially read a lot of characters into the buffer, and we may
+        // not have processed all of them. Therefore, we need to "rewind" the
+        // stream after the last processed char.
+        is.seekg(start_position);
+        is.ignore(static_cast<std::streamsize>(processed_chars));
+        // clear stream flags
+        is.clear();
+    }
+
+    int get_character() override
+    {
+        // check if refilling is necessary and possible
+        if (buffer_pos == fill_size and not eof)
+        {
+            fill_buffer();
+
+            // check and remember that filling did not yield new input
+            if (fill_size == 0)
+            {
+                eof = true;
+                return std::char_traits<char>::eof();
+            }
+
+            // the buffer is ready
+            buffer_pos = 0;
+        }
+
+        ++processed_chars;
+        assert(buffer_pos < buffer.size());
+        return buffer[buffer_pos++] & 0xFF;
+    }
+
+    std::string read(std::size_t offset, std::size_t length) override
+    {
+        // create buffer
+        std::string result(length, '\0');
+
+        // save stream position
+        const auto current_pos = is.tellg();
+        // save stream flags
+        const auto flags = is.rdstate();
+
+        // clear stream flags
+        is.clear();
+        // set stream position
+        is.seekg(static_cast<std::streamoff>(offset));
+        // read bytes
+        is.read(&result[0], static_cast<std::streamsize>(length));
+
+        // reset stream position
+        is.seekg(current_pos);
+        // reset stream flags
+        is.setstate(flags);
+
+        return result;
+    }
+
+  private:
+    void fill_buffer()
+    {
+        // fill
+        is.read(buffer.data(), static_cast<std::streamsize>(buffer.size()));
+        // store number of bytes in the buffer
+        fill_size = static_cast<size_t>(is.gcount());
+    }
+
+    /// the associated input stream
+    std::istream& is;
+
+    /// chars returned via get_character()
+    std::size_t processed_chars = 0;
+    /// chars processed in the current buffer
+    std::size_t buffer_pos = 0;
+
+    /// whether stream reached eof
+    bool eof = false;
+    /// how many chars have been copied to the buffer by last (re)fill
+    std::size_t fill_size = 0;
+
+    /// position of the stream when we started
+    const std::streampos start_position;
+
+    /// internal buffer
+    std::array<char, BufferSize> buffer{{}};
+};
+
+/// input adapter for buffer input
+class input_buffer_adapter : public input_adapter_protocol
+{
+  public:
+    input_buffer_adapter(const char* b, const std::size_t l)
+        : cursor(b), limit(b + l), start(b)
+    {
+        // skip byte order mark
+        if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF')
+        {
+            cursor += 3;
+        }
+    }
+
+    // delete because of pointer members
+    input_buffer_adapter(const input_buffer_adapter&) = delete;
+    input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
+
+    int get_character() noexcept override
+    {
+        if (JSON_LIKELY(cursor < limit))
+        {
+            return *(cursor++) & 0xFF;
+        }
+
+        return std::char_traits<char>::eof();
+    }
+
+    std::string read(std::size_t offset, std::size_t length) override
+    {
+        // avoid reading too many characters
+        const auto max_length = static_cast<size_t>(limit - start);
+        return std::string(start + offset, (std::min)(length, max_length - offset));
+    }
+
+  private:
+    /// pointer to the current character
+    const char* cursor;
+    /// pointer past the last character
+    const char* limit;
+    /// pointer to the first character
+    const char* start;
+};
+
+class input_adapter
+{
+  public:
+    // native support
+
+    /// input adapter for input stream
+    input_adapter(std::istream& i)
+        : ia(std::make_shared<cached_input_stream_adapter<16384>>(i)) {}
+
+    /// input adapter for input stream
+    input_adapter(std::istream&& i)
+        : ia(std::make_shared<cached_input_stream_adapter<16384>>(i)) {}
+
+    /// input adapter for buffer
+    template<typename CharT,
+             typename std::enable_if<
+                 std::is_pointer<CharT>::value and
+                 std::is_integral<
+                     typename std::remove_pointer<CharT>::type>::value and
+                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
+                 int>::type = 0>
+    input_adapter(CharT b, std::size_t l)
+        : ia(std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(b), l)) {}
+
+    // derived support
+
+    /// input adapter for string literal
+    template<typename CharT,
+             typename std::enable_if<
+                 std::is_pointer<CharT>::value and
+                 std::is_integral<
+                     typename std::remove_pointer<CharT>::type>::value and
+                 sizeof(typename std::remove_pointer<CharT>::type) == 1,
+                 int>::type = 0>
+    input_adapter(CharT b)
+        : input_adapter(reinterpret_cast<const char*>(b),
+                        std::strlen(reinterpret_cast<const char*>(b))) {}
+
+    /// input adapter for iterator range with contiguous storage
+    template<class IteratorType,
+             typename std::enable_if<
+                 std::is_same<typename std::iterator_traits<IteratorType>::iterator_category,
+                              std::random_access_iterator_tag>::value,
+                 int>::type = 0>
+    input_adapter(IteratorType first, IteratorType last)
+    {
+        // assertion to check that the iterator range is indeed contiguous,
+        // see http://stackoverflow.com/a/35008842/266378 for more discussion
+        assert(std::accumulate(
+                   first, last, std::pair<bool, int>(true, 0),
+                   [&first](std::pair<bool, int> res, decltype(*first) val)
+        {
+            res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
+            return res;
+        }).first);
+
+        // assertion to check that each element is 1 byte long
+        static_assert(
+            sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
+            "each element in the iterator range must have the size of 1 byte");
+
+        const auto len = static_cast<size_t>(std::distance(first, last));
+        if (JSON_LIKELY(len > 0))
+        {
+            // there is at least one element: use the address of first
+            ia = std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(&(*first)), len);
+        }
+        else
+        {
+            // the address of first cannot be used: use nullptr
+            ia = std::make_shared<input_buffer_adapter>(nullptr, len);
+        }
+    }
+
+    /// input adapter for array
+    template<class T, std::size_t N>
+    input_adapter(T (&array)[N])
+        : input_adapter(std::begin(array), std::end(array)) {}
+
+    /// input adapter for contiguous container
+    template <
+        class ContiguousContainer,
+        typename std::enable_if <
+            not std::is_pointer<ContiguousContainer>::value and
+            std::is_base_of<std::random_access_iterator_tag,
+                            typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value,
+            int >::type = 0 >
+    input_adapter(const ContiguousContainer& c)
+        : input_adapter(std::begin(c), std::end(c)) {}
+
+    operator input_adapter_t()
+    {
+        return ia;
+    }
+
+  private:
+    /// the actual adapter
+    input_adapter_t ia = nullptr;
+};
+
+//////////////////////
+// lexer and parser //
+//////////////////////
+
+/*!
+@brief lexical analysis
+
+This class organizes the lexical analysis during JSON deserialization.
+*/
+template<typename BasicJsonType>
+class lexer
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+
+  public:
+    /// token types for the parser
+    enum class token_type
+    {
+        uninitialized,    ///< indicating the scanner is uninitialized
+        literal_true,     ///< the `true` literal
+        literal_false,    ///< the `false` literal
+        literal_null,     ///< the `null` literal
+        value_string,     ///< a string -- use get_string() for actual value
+        value_unsigned,   ///< an unsigned integer -- use get_number_unsigned() for actual value
+        value_integer,    ///< a signed integer -- use get_number_integer() for actual value
+        value_float,      ///< an floating point number -- use get_number_float() for actual value
+        begin_array,      ///< the character for array begin `[`
+        begin_object,     ///< the character for object begin `{`
+        end_array,        ///< the character for array end `]`
+        end_object,       ///< the character for object end `}`
+        name_separator,   ///< the name separator `:`
+        value_separator,  ///< the value separator `,`
+        parse_error,      ///< indicating a parse error
+        end_of_input,     ///< indicating the end of the input buffer
+        literal_or_value  ///< a literal or the begin of a value (only for diagnostics)
+    };
+
+    /// return name of values of type token_type (only used for errors)
+    static const char* token_type_name(const token_type t) noexcept
+    {
+        switch (t)
+        {
+            case token_type::uninitialized:
+                return "<uninitialized>";
+            case token_type::literal_true:
+                return "true literal";
+            case token_type::literal_false:
+                return "false literal";
+            case token_type::literal_null:
+                return "null literal";
+            case token_type::value_string:
+                return "string literal";
+            case lexer::token_type::value_unsigned:
+            case lexer::token_type::value_integer:
+            case lexer::token_type::value_float:
+                return "number literal";
+            case token_type::begin_array:
+                return "'['";
+            case token_type::begin_object:
+                return "'{'";
+            case token_type::end_array:
+                return "']'";
+            case token_type::end_object:
+                return "'}'";
+            case token_type::name_separator:
+                return "':'";
+            case token_type::value_separator:
+                return "','";
+            case token_type::parse_error:
+                return "<parse error>";
+            case token_type::end_of_input:
+                return "end of input";
+            case token_type::literal_or_value:
+                return "'[', '{', or a literal";
+            default: // catch non-enum values
+                return "unknown token"; // LCOV_EXCL_LINE
+        }
+    }
+
+    explicit lexer(detail::input_adapter_t adapter)
+        : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {}
+
+    // delete because of pointer members
+    lexer(const lexer&) = delete;
+    lexer& operator=(lexer&) = delete;
+
+  private:
+    /////////////////////
+    // locales
+    /////////////////////
+
+    /// return the locale-dependent decimal point
+    static char get_decimal_point() noexcept
+    {
+        const auto loc = localeconv();
+        assert(loc != nullptr);
+        return (loc->decimal_point == nullptr) ? '.' : loc->decimal_point[0];
+    }
+
+    /////////////////////
+    // scan functions
+    /////////////////////
+
+    /*!
+    @brief get codepoint from 4 hex characters following `\u`
+
+    For input "\u c1 c2 c3 c4" the codepoint is:
+      (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4
+    = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0)
+
+    Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f'
+    must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The
+    conversion is done by subtracting the offset (0x30, 0x37, and 0x57)
+    between the ASCII value of the character and the desired integer value.
+
+    @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or
+            non-hex character)
+    */
+    int get_codepoint()
+    {
+        // this function only makes sense after reading `\u`
+        assert(current == 'u');
+        int codepoint = 0;
+
+        const auto factors = { 12, 8, 4, 0 };
+        for (const auto factor : factors)
+        {
+            get();
+
+            if (current >= '0' and current <= '9')
+            {
+                codepoint += ((current - 0x30) << factor);
+            }
+            else if (current >= 'A' and current <= 'F')
+            {
+                codepoint += ((current - 0x37) << factor);
+            }
+            else if (current >= 'a' and current <= 'f')
+            {
+                codepoint += ((current - 0x57) << factor);
+            }
+            else
+            {
+                return -1;
+            }
+        }
+
+        assert(0x0000 <= codepoint and codepoint <= 0xFFFF);
+        return codepoint;
+    }
+
+    /*!
+    @brief check if the next byte(s) are inside a given range
+
+    Adds the current byte and, for each passed range, reads a new byte and
+    checks if it is inside the range. If a violation was detected, set up an
+    error message and return false. Otherwise, return true.
+
+    @return true if and only if no range violation was detected
+    */
+    bool next_byte_in_range(std::initializer_list<int> ranges)
+    {
+        assert(ranges.size() == 2 or ranges.size() == 4 or ranges.size() == 6);
+        add(current);
+
+        for (auto range = ranges.begin(); range != ranges.end(); ++range)
+        {
+            get();
+            if (JSON_LIKELY(*range <= current and current <= *(++range)))
+            {
+                add(current);
+            }
+            else
+            {
+                error_message = "invalid string: ill-formed UTF-8 byte";
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /*!
+    @brief scan a string literal
+
+    This function scans a string according to Sect. 7 of RFC 7159. While
+    scanning, bytes are escaped and copied into buffer yytext. Then the
+    function returns successfully, yytext is null-terminated and yylen
+    contains the number of bytes in the string.
+
+    @return token_type::value_string if string could be successfully scanned,
+            token_type::parse_error otherwise
+
+    @note In case of errors, variable error_message contains a textual
+          description.
+    */
+    token_type scan_string()
+    {
+        // reset yytext (ignore opening quote)
+        reset();
+
+        // we entered the function by reading an open quote
+        assert(current == '\"');
+
+        while (true)
+        {
+            // get next character
+            switch (get())
+            {
+                // end of file while parsing string
+                case std::char_traits<char>::eof():
+                {
+                    error_message = "invalid string: missing closing quote";
+                    return token_type::parse_error;
+                }
+
+                // closing quote
+                case '\"':
+                {
+                    // terminate yytext
+                    add('\0');
+                    --yylen;
+                    return token_type::value_string;
+                }
+
+                // escapes
+                case '\\':
+                {
+                    switch (get())
+                    {
+                        // quotation mark
+                        case '\"':
+                            add('\"');
+                            break;
+                        // reverse solidus
+                        case '\\':
+                            add('\\');
+                            break;
+                        // solidus
+                        case '/':
+                            add('/');
+                            break;
+                        // backspace
+                        case 'b':
+                            add('\b');
+                            break;
+                        // form feed
+                        case 'f':
+                            add('\f');
+                            break;
+                        // line feed
+                        case 'n':
+                            add('\n');
+                            break;
+                        // carriage return
+                        case 'r':
+                            add('\r');
+                            break;
+                        // tab
+                        case 't':
+                            add('\t');
+                            break;
+
+                        // unicode escapes
+                        case 'u':
+                        {
+                            int codepoint;
+                            const int codepoint1 = get_codepoint();
+
+                            if (JSON_UNLIKELY(codepoint1 == -1))
+                            {
+                                error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                return token_type::parse_error;
+                            }
+
+                            // check if code point is a high surrogate
+                            if (0xD800 <= codepoint1 and codepoint1 <= 0xDBFF)
+                            {
+                                // expect next \uxxxx entry
+                                if (JSON_LIKELY(get() == '\\' and get() == 'u'))
+                                {
+                                    const int codepoint2 = get_codepoint();
+
+                                    if (JSON_UNLIKELY(codepoint2 == -1))
+                                    {
+                                        error_message = "invalid string: '\\u' must be followed by 4 hex digits";
+                                        return token_type::parse_error;
+                                    }
+
+                                    // check if codepoint2 is a low surrogate
+                                    if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF))
+                                    {
+                                        codepoint =
+                                            // high surrogate occupies the most significant 22 bits
+                                            (codepoint1 << 10)
+                                            // low surrogate occupies the least significant 15 bits
+                                            + codepoint2
+                                            // there is still the 0xD800, 0xDC00 and 0x10000 noise
+                                            // in the result so we have to subtract with:
+                                            // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
+                                            - 0x35FDC00;
+                                    }
+                                    else
+                                    {
+                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                        return token_type::parse_error;
+                                    }
+                                }
+                                else
+                                {
+                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
+                                    return token_type::parse_error;
+                                }
+                            }
+                            else
+                            {
+                                if (JSON_UNLIKELY(0xDC00 <= codepoint1 and codepoint1 <= 0xDFFF))
+                                {
+                                    error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
+                                    return token_type::parse_error;
+                                }
+
+                                // only work with first code point
+                                codepoint = codepoint1;
+                            }
+
+                            // result of the above calculation yields a proper codepoint
+                            assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
+
+                            // translate code point to bytes
+                            if (codepoint < 0x80)
+                            {
+                                // 1-byte characters: 0xxxxxxx (ASCII)
+                                add(codepoint);
+                            }
+                            else if (codepoint <= 0x7ff)
+                            {
+                                // 2-byte characters: 110xxxxx 10xxxxxx
+                                add(0xC0 | (codepoint >> 6));
+                                add(0x80 | (codepoint & 0x3F));
+                            }
+                            else if (codepoint <= 0xffff)
+                            {
+                                // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
+                                add(0xE0 | (codepoint >> 12));
+                                add(0x80 | ((codepoint >> 6) & 0x3F));
+                                add(0x80 | (codepoint & 0x3F));
+                            }
+                            else
+                            {
+                                // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+                                add(0xF0 | (codepoint >> 18));
+                                add(0x80 | ((codepoint >> 12) & 0x3F));
+                                add(0x80 | ((codepoint >> 6) & 0x3F));
+                                add(0x80 | (codepoint & 0x3F));
+                            }
+
+                            break;
+                        }
+
+                        // other characters after escape
+                        default:
+                            error_message = "invalid string: forbidden character after backslash";
+                            return token_type::parse_error;
+                    }
+
+                    break;
+                }
+
+                // invalid control characters
+                case 0x00:
+                case 0x01:
+                case 0x02:
+                case 0x03:
+                case 0x04:
+                case 0x05:
+                case 0x06:
+                case 0x07:
+                case 0x08:
+                case 0x09:
+                case 0x0a:
+                case 0x0b:
+                case 0x0c:
+                case 0x0d:
+                case 0x0e:
+                case 0x0f:
+                case 0x10:
+                case 0x11:
+                case 0x12:
+                case 0x13:
+                case 0x14:
+                case 0x15:
+                case 0x16:
+                case 0x17:
+                case 0x18:
+                case 0x19:
+                case 0x1a:
+                case 0x1b:
+                case 0x1c:
+                case 0x1d:
+                case 0x1e:
+                case 0x1f:
+                {
+                    error_message = "invalid string: control character must be escaped";
+                    return token_type::parse_error;
+                }
+
+                // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))
+                case 0x20:
+                case 0x21:
+                case 0x23:
+                case 0x24:
+                case 0x25:
+                case 0x26:
+                case 0x27:
+                case 0x28:
+                case 0x29:
+                case 0x2a:
+                case 0x2b:
+                case 0x2c:
+                case 0x2d:
+                case 0x2e:
+                case 0x2f:
+                case 0x30:
+                case 0x31:
+                case 0x32:
+                case 0x33:
+                case 0x34:
+                case 0x35:
+                case 0x36:
+                case 0x37:
+                case 0x38:
+                case 0x39:
+                case 0x3a:
+                case 0x3b:
+                case 0x3c:
+                case 0x3d:
+                case 0x3e:
+                case 0x3f:
+                case 0x40:
+                case 0x41:
+                case 0x42:
+                case 0x43:
+                case 0x44:
+                case 0x45:
+                case 0x46:
+                case 0x47:
+                case 0x48:
+                case 0x49:
+                case 0x4a:
+                case 0x4b:
+                case 0x4c:
+                case 0x4d:
+                case 0x4e:
+                case 0x4f:
+                case 0x50:
+                case 0x51:
+                case 0x52:
+                case 0x53:
+                case 0x54:
+                case 0x55:
+                case 0x56:
+                case 0x57:
+                case 0x58:
+                case 0x59:
+                case 0x5a:
+                case 0x5b:
+                case 0x5d:
+                case 0x5e:
+                case 0x5f:
+                case 0x60:
+                case 0x61:
+                case 0x62:
+                case 0x63:
+                case 0x64:
+                case 0x65:
+                case 0x66:
+                case 0x67:
+                case 0x68:
+                case 0x69:
+                case 0x6a:
+                case 0x6b:
+                case 0x6c:
+                case 0x6d:
+                case 0x6e:
+                case 0x6f:
+                case 0x70:
+                case 0x71:
+                case 0x72:
+                case 0x73:
+                case 0x74:
+                case 0x75:
+                case 0x76:
+                case 0x77:
+                case 0x78:
+                case 0x79:
+                case 0x7a:
+                case 0x7b:
+                case 0x7c:
+                case 0x7d:
+                case 0x7e:
+                case 0x7f:
+                {
+                    add(current);
+                    break;
+                }
+
+                // U+0080..U+07FF: bytes C2..DF 80..BF
+                case 0xc2:
+                case 0xc3:
+                case 0xc4:
+                case 0xc5:
+                case 0xc6:
+                case 0xc7:
+                case 0xc8:
+                case 0xc9:
+                case 0xca:
+                case 0xcb:
+                case 0xcc:
+                case 0xcd:
+                case 0xce:
+                case 0xcf:
+                case 0xd0:
+                case 0xd1:
+                case 0xd2:
+                case 0xd3:
+                case 0xd4:
+                case 0xd5:
+                case 0xd6:
+                case 0xd7:
+                case 0xd8:
+                case 0xd9:
+                case 0xda:
+                case 0xdb:
+                case 0xdc:
+                case 0xdd:
+                case 0xde:
+                case 0xdf:
+                {
+                    if (JSON_UNLIKELY(not next_byte_in_range({0x80, 0xBF})))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
+                case 0xe0:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
+                // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
+                case 0xe1:
+                case 0xe2:
+                case 0xe3:
+                case 0xe4:
+                case 0xe5:
+                case 0xe6:
+                case 0xe7:
+                case 0xe8:
+                case 0xe9:
+                case 0xea:
+                case 0xeb:
+                case 0xec:
+                case 0xee:
+                case 0xef:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+D000..U+D7FF: bytes ED 80..9F 80..BF
+                case 0xed:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x9F, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
+                case 0xf0:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
+                case 0xf1:
+                case 0xf2:
+                case 0xf3:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
+                case 0xf4:
+                {
+                    if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF}))))
+                    {
+                        return token_type::parse_error;
+                    }
+                    break;
+                }
+
+                // remaining bytes (80..C1 and F5..FF) are ill-formed
+                default:
+                {
+                    error_message = "invalid string: ill-formed UTF-8 byte";
+                    return token_type::parse_error;
+                }
+            }
+        }
+    }
+
+    static void strtof(float& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtof(str, endptr);
+    }
+
+    static void strtof(double& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtod(str, endptr);
+    }
+
+    static void strtof(long double& f, const char* str, char** endptr) noexcept
+    {
+        f = std::strtold(str, endptr);
+    }
+
+    /*!
+    @brief scan a number literal
+
+    This function scans a string according to Sect. 6 of RFC 7159.
+
+    The function is realized with a deterministic finite state machine derived
+    from the grammar described in RFC 7159. Starting in state "init", the
+    input is read and used to determined the next state. Only state "done"
+    accepts the number. State "error" is a trap state to model errors. In the
+    table below, "anything" means any character but the ones listed before.
+
+    state    | 0        | 1-9      | e E      | +       | -       | .        | anything
+    ---------|----------|----------|----------|---------|---------|----------|-----------
+    init     | zero     | any1     | [error]  | [error] | minus   | [error]  | [error]
+    minus    | zero     | any1     | [error]  | [error] | [error] | [error]  | [error]
+    zero     | done     | done     | exponent | done    | done    | decimal1 | done
+    any1     | any1     | any1     | exponent | done    | done    | decimal1 | done
+    decimal1 | decimal2 | [error]  | [error]  | [error] | [error] | [error]  | [error]
+    decimal2 | decimal2 | decimal2 | exponent | done    | done    | done     | done
+    exponent | any2     | any2     | [error]  | sign    | sign    | [error]  | [error]
+    sign     | any2     | any2     | [error]  | [error] | [error] | [error]  | [error]
+    any2     | any2     | any2     | done     | done    | done    | done     | done
+
+    The state machine is realized with one label per state (prefixed with
+    "scan_number_") and `goto` statements between them. The state machine
+    contains cycles, but any cycle can be left when EOF is read. Therefore,
+    the function is guaranteed to terminate.
+
+    During scanning, the read bytes are stored in yytext. This string is
+    then converted to a signed integer, an unsigned integer, or a
+    floating-point number.
+
+    @return token_type::value_unsigned, token_type::value_integer, or
+            token_type::value_float if number could be successfully scanned,
+            token_type::parse_error otherwise
+
+    @note The scanner is independent of the current locale. Internally, the
+          locale's decimal point is used instead of `.` to work with the
+          locale-dependent converters.
+    */
+    token_type scan_number()
+    {
+        // reset yytext to store the number's bytes
+        reset();
+
+        // the type of the parsed number; initially set to unsigned; will be
+        // changed if minus sign, decimal point or exponent is read
+        token_type number_type = token_type::value_unsigned;
+
+        // state (init): we just found out we need to scan a number
+        switch (current)
+        {
+            case '-':
+            {
+                add(current);
+                goto scan_number_minus;
+            }
+
+            case '0':
+            {
+                add(current);
+                goto scan_number_zero;
+            }
+
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            default:
+            {
+                // all other characters are rejected outside scan_number()
+                assert(false); // LCOV_EXCL_LINE
+            }
+        }
+
+scan_number_minus:
+        // state: we just parsed a leading minus sign
+        number_type = token_type::value_integer;
+        switch (get())
+        {
+            case '0':
+            {
+                add(current);
+                goto scan_number_zero;
+            }
+
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after '-'";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_zero:
+        // state: we just parse a zero (maybe with a leading minus sign)
+        switch (get())
+        {
+            case '.':
+            {
+                add(decimal_point_char);
+                goto scan_number_decimal1;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_any1:
+        // state: we just parsed a number 0-9 (maybe with a leading minus sign)
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any1;
+            }
+
+            case '.':
+            {
+                add(decimal_point_char);
+                goto scan_number_decimal1;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_decimal1:
+        // state: we just parsed a decimal point
+        number_type = token_type::value_float;
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_decimal2;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after '.'";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_decimal2:
+        // we just parsed at least one number after a decimal point
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_decimal2;
+            }
+
+            case 'e':
+            case 'E':
+            {
+                add(current);
+                goto scan_number_exponent;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_exponent:
+        // we just parsed an exponent
+        number_type = token_type::value_float;
+        switch (get())
+        {
+            case '+':
+            case '-':
+            {
+                add(current);
+                goto scan_number_sign;
+            }
+
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+            {
+                error_message =
+                    "invalid number; expected '+', '-', or digit after exponent";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_sign:
+        // we just parsed an exponent sign
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+            {
+                error_message = "invalid number; expected digit after exponent sign";
+                return token_type::parse_error;
+            }
+        }
+
+scan_number_any2:
+        // we just parsed a number after the exponent or exponent sign
+        switch (get())
+        {
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+            {
+                add(current);
+                goto scan_number_any2;
+            }
+
+            default:
+                goto scan_number_done;
+        }
+
+scan_number_done:
+        // unget the character after the number (we only read it to know that
+        // we are done scanning a number)
+        --chars_read;
+        next_unget = true;
+
+        // terminate token
+        add('\0');
+        --yylen;
+
+        char* endptr = nullptr;
+        errno = 0;
+
+        // try to parse integers first and fall back to floats
+        if (number_type == token_type::value_unsigned)
+        {
+            const auto x = std::strtoull(yytext.data(), &endptr, 10);
+
+            // we checked the number format before
+            assert(endptr == yytext.data() + yylen);
+
+            if (errno == 0)
+            {
+                value_unsigned = static_cast<number_unsigned_t>(x);
+                if (value_unsigned == x)
+                {
+                    return token_type::value_unsigned;
+                }
+            }
+        }
+        else if (number_type == token_type::value_integer)
+        {
+            const auto x = std::strtoll(yytext.data(), &endptr, 10);
+
+            // we checked the number format before
+            assert(endptr == yytext.data() + yylen);
+
+            if (errno == 0)
+            {
+                value_integer = static_cast<number_integer_t>(x);
+                if (value_integer == x)
+                {
+                    return token_type::value_integer;
+                }
+            }
+        }
+
+        // this code is reached if we parse a floating-point number or if an
+        // integer conversion above failed
+        strtof(value_float, yytext.data(), &endptr);
+
+        // we checked the number format before
+        assert(endptr == yytext.data() + yylen);
+
+        return token_type::value_float;
+    }
+
+    /*!
+    @param[in] literal_text  the literal text to expect
+    @param[in] length        the length of the passed literal text
+    @param[in] return_type   the token type to return on success
+    */
+    token_type scan_literal(const char* literal_text, const std::size_t length,
+                            token_type return_type)
+    {
+        assert(current == literal_text[0]);
+        for (std::size_t i = 1; i < length; ++i)
+        {
+            if (JSON_UNLIKELY(get() != literal_text[i]))
+            {
+                error_message = "invalid literal";
+                return token_type::parse_error;
+            }
+        }
+        return return_type;
+    }
+
+    /////////////////////
+    // input management
+    /////////////////////
+
+    /// reset yytext
+    void reset() noexcept
+    {
+        yylen = 0;
+        start_pos = chars_read - 1;
+    }
+
+    /// get a character from the input
+    int get()
+    {
+        ++chars_read;
+        return next_unget ? (next_unget = false, current)
+               : (current = ia->get_character());
+    }
+
+    /// add a character to yytext
+    void add(int c)
+    {
+        // resize yytext if necessary; this condition is deemed unlikely,
+        // because we start with a 1024-byte buffer
+        if (JSON_UNLIKELY((yylen + 1 > yytext.capacity())))
+        {
+            yytext.resize(2 * yytext.capacity(), '\0');
+        }
+        assert(yylen < yytext.size());
+        yytext[yylen++] = static_cast<char>(c);
+    }
+
+  public:
+    /////////////////////
+    // value getters
+    /////////////////////
+
+    /// return integer value
+    constexpr number_integer_t get_number_integer() const noexcept
+    {
+        return value_integer;
+    }
+
+    /// return unsigned integer value
+    constexpr number_unsigned_t get_number_unsigned() const noexcept
+    {
+        return value_unsigned;
+    }
+
+    /// return floating-point value
+    constexpr number_float_t get_number_float() const noexcept
+    {
+        return value_float;
+    }
+
+    /// return string value
+    const std::string get_string()
+    {
+        // yytext cannot be returned as char*, because it may contain a null
+        // byte (parsed as "\u0000")
+        return std::string(yytext.data(), yylen);
+    }
+
+    /////////////////////
+    // diagnostics
+    /////////////////////
+
+    /// return position of last read token
+    constexpr std::size_t get_position() const noexcept
+    {
+        return chars_read;
+    }
+
+    /// return the last read token (for errors only)
+    std::string get_token_string() const
+    {
+        // get the raw byte sequence of the last token
+        std::string s = ia->read(start_pos, chars_read - start_pos);
+
+        // escape control characters
+        std::string result;
+        for (auto c : s)
+        {
+            if (c == '\0' or c == std::char_traits<char>::eof())
+            {
+                // ignore EOF
+                continue;
+            }
+            else if ('\x00' <= c and c <= '\x1f')
+            {
+                // escape control characters
+                std::stringstream ss;
+                ss << "<U+" << std::setw(4) << std::uppercase << std::setfill('0')
+                   << std::hex << static_cast<int>(c) << ">";
+                result += ss.str();
+            }
+            else
+            {
+                // add character as is
+                result.push_back(c);
+            }
+        }
+
+        return result;
+    }
+
+    /// return syntax error message
+    constexpr const char* get_error_message() const noexcept
+    {
+        return error_message;
+    }
+
+    /////////////////////
+    // actual scanner
+    /////////////////////
+
+    token_type scan()
+    {
+        // read next character and ignore whitespace
+        do
+        {
+            get();
+        }
+        while (current == ' ' or current == '\t' or current == '\n' or current == '\r');
+
+        switch (current)
+        {
+            // structural characters
+            case '[':
+                return token_type::begin_array;
+            case ']':
+                return token_type::end_array;
+            case '{':
+                return token_type::begin_object;
+            case '}':
+                return token_type::end_object;
+            case ':':
+                return token_type::name_separator;
+            case ',':
+                return token_type::value_separator;
+
+            // literals
+            case 't':
+                return scan_literal("true", 4, token_type::literal_true);
+            case 'f':
+                return scan_literal("false", 5, token_type::literal_false);
+            case 'n':
+                return scan_literal("null", 4, token_type::literal_null);
+
+            // string
+            case '\"':
+                return scan_string();
+
+            // number
+            case '-':
+            case '0':
+            case '1':
+            case '2':
+            case '3':
+            case '4':
+            case '5':
+            case '6':
+            case '7':
+            case '8':
+            case '9':
+                return scan_number();
+
+            // end of input (the null byte is needed when parsing from
+            // string literals)
+            case '\0':
+            case std::char_traits<char>::eof():
+                return token_type::end_of_input;
+
+            // error
+            default:
+                error_message = "invalid literal";
+                return token_type::parse_error;
+        }
+    }
+
+  private:
+    /// input adapter
+    detail::input_adapter_t ia = nullptr;
+
+    /// the current character
+    int current = std::char_traits<char>::eof();
+
+    /// whether get() should return the last character again
+    bool next_unget = false;
+
+    /// the number of characters read
+    std::size_t chars_read = 0;
+    /// the start position of the current token
+    std::size_t start_pos = 0;
+
+    /// buffer for variable-length tokens (numbers, strings)
+    std::vector<char> yytext = std::vector<char>(1024, '\0');
+    /// current index in yytext
+    std::size_t yylen = 0;
+
+    /// a description of occurred lexer errors
+    const char* error_message = "";
+
+    // number values
+    number_integer_t value_integer = 0;
+    number_unsigned_t value_unsigned = 0;
+    number_float_t value_float = 0;
+
+    /// the decimal point
+    const char decimal_point_char = '.';
+};
+
+/*!
+@brief syntax analysis
+
+This class implements a recursive decent parser.
+*/
+template<typename BasicJsonType>
+class parser
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using lexer_t = lexer<BasicJsonType>;
+    using token_type = typename lexer_t::token_type;
+
+  public:
+    enum class parse_event_t : uint8_t
+    {
+        /// the parser read `{` and started to process a JSON object
+        object_start,
+        /// the parser read `}` and finished processing a JSON object
+        object_end,
+        /// the parser read `[` and started to process a JSON array
+        array_start,
+        /// the parser read `]` and finished processing a JSON array
+        array_end,
+        /// the parser read a key of a value in an object
+        key,
+        /// the parser finished reading a JSON value
+        value
+    };
+
+    using parser_callback_t =
+        std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>;
+
+    /// a parser reading from an input adapter
+    explicit parser(detail::input_adapter_t adapter,
+                    const parser_callback_t cb = nullptr,
+                    const bool allow_exceptions_ = true)
+        : callback(cb), m_lexer(adapter), allow_exceptions(allow_exceptions_)
+    {}
+
+    /*!
+    @brief public parser interface
+
+    @param[in] strict      whether to expect the last token to be EOF
+    @param[in,out] result  parsed JSON value
+
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+    */
+    void parse(const bool strict, BasicJsonType& result)
+    {
+        // read first token
+        get_token();
+
+        parse_internal(true, result);
+        result.assert_invariant();
+
+        // in strict mode, input must be completely read
+        if (strict)
+        {
+            get_token();
+            expect(token_type::end_of_input);
+        }
+
+        // in case of an error, return discarded value
+        if (errored)
+        {
+            result = value_t::discarded;
+            return;
+        }
+
+        // set top-level value to null if it was discarded by the callback
+        // function
+        if (result.is_discarded())
+        {
+            result = nullptr;
+        }
+    }
+
+    /*!
+    @brief public accept interface
+
+    @param[in] strict  whether to expect the last token to be EOF
+    @return whether the input is a proper JSON text
+    */
+    bool accept(const bool strict = true)
+    {
+        // read first token
+        get_token();
+
+        if (not accept_internal())
+        {
+            return false;
+        }
+
+        // strict => last token must be EOF
+        return not strict or (get_token() == token_type::end_of_input);
+    }
+
+  private:
+    /*!
+    @brief the actual parser
+    @throw parse_error.101 in case of an unexpected token
+    @throw parse_error.102 if to_unicode fails or surrogate error
+    @throw parse_error.103 if to_unicode fails
+    */
+    void parse_internal(bool keep, BasicJsonType& result)
+    {
+        // never parse after a parse error was detected
+        assert(not errored);
+
+        // start with a discarded value
+        if (not result.is_discarded())
+        {
+            result.m_value.destroy(result.m_type);
+            result.m_type = value_t::discarded;
+        }
+
+        switch (last_token)
+        {
+            case token_type::begin_object:
+            {
+                if (keep and (not callback or ((keep = callback(depth++, parse_event_t::object_start, result)))))
+                {
+                    // explicitly set result to object to cope with {}
+                    result.m_type = value_t::object;
+                    result.m_value = value_t::object;
+                }
+
+                // read next token
+                get_token();
+
+                // closing } -> we are done
+                if (last_token == token_type::end_object)
+                {
+                    if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
+                    {
+                        result.m_value.destroy(result.m_type);
+                        result.m_type = value_t::discarded;
+                    }
+                    break;
+                }
+
+                // parse values
+                std::string key;
+                BasicJsonType value;
+                while (true)
+                {
+                    // store key
+                    if (not expect(token_type::value_string))
+                    {
+                        return;
+                    }
+                    key = m_lexer.get_string();
+
+                    bool keep_tag = false;
+                    if (keep)
+                    {
+                        if (callback)
+                        {
+                            BasicJsonType k(key);
+                            keep_tag = callback(depth, parse_event_t::key, k);
+                        }
+                        else
+                        {
+                            keep_tag = true;
+                        }
+                    }
+
+                    // parse separator (:)
+                    get_token();
+                    if (not expect(token_type::name_separator))
+                    {
+                        return;
+                    }
+
+                    // parse and add value
+                    get_token();
+                    value.m_value.destroy(value.m_type);
+                    value.m_type = value_t::discarded;
+                    parse_internal(keep, value);
+
+                    if (JSON_UNLIKELY(errored))
+                    {
+                        return;
+                    }
+
+                    if (keep and keep_tag and not value.is_discarded())
+                    {
+                        result.m_value.object->emplace(std::move(key), std::move(value));
+                    }
+
+                    // comma -> next value
+                    get_token();
+                    if (last_token == token_type::value_separator)
+                    {
+                        get_token();
+                        continue;
+                    }
+
+                    // closing }
+                    if (not expect(token_type::end_object))
+                    {
+                        return;
+                    }
+                    break;
+                }
+
+                if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
+                {
+                    result.m_value.destroy(result.m_type);
+                    result.m_type = value_t::discarded;
+                }
+                break;
+            }
+
+            case token_type::begin_array:
+            {
+                if (keep and (not callback or ((keep = callback(depth++, parse_event_t::array_start, result)))))
+                {
+                    // explicitly set result to object to cope with []
+                    result.m_type = value_t::array;
+                    result.m_value = value_t::array;
+                }
+
+                // read next token
+                get_token();
+
+                // closing ] -> we are done
+                if (last_token == token_type::end_array)
+                {
+                    if (callback and not callback(--depth, parse_event_t::array_end, result))
+                    {
+                        result.m_value.destroy(result.m_type);
+                        result.m_type = value_t::discarded;
+                    }
+                    break;
+                }
+
+                // parse values
+                BasicJsonType value;
+                while (true)
+                {
+                    // parse value
+                    value.m_value.destroy(value.m_type);
+                    value.m_type = value_t::discarded;
+                    parse_internal(keep, value);
+
+                    if (JSON_UNLIKELY(errored))
+                    {
+                        return;
+                    }
+
+                    if (keep and not value.is_discarded())
+                    {
+                        result.m_value.array->push_back(std::move(value));
+                    }
+
+                    // comma -> next value
+                    get_token();
+                    if (last_token == token_type::value_separator)
+                    {
+                        get_token();
+                        continue;
+                    }
+
+                    // closing ]
+                    if (not expect(token_type::end_array))
+                    {
+                        return;
+                    }
+                    break;
+                }
+
+                if (keep and callback and not callback(--depth, parse_event_t::array_end, result))
+                {
+                    result.m_value.destroy(result.m_type);
+                    result.m_type = value_t::discarded;
+                }
+                break;
+            }
+
+            case token_type::literal_null:
+            {
+                result.m_type = value_t::null;
+                break;
+            }
+
+            case token_type::value_string:
+            {
+                result.m_type = value_t::string;
+                result.m_value = m_lexer.get_string();
+                break;
+            }
+
+            case token_type::literal_true:
+            {
+                result.m_type = value_t::boolean;
+                result.m_value = true;
+                break;
+            }
+
+            case token_type::literal_false:
+            {
+                result.m_type = value_t::boolean;
+                result.m_value = false;
+                break;
+            }
+
+            case token_type::value_unsigned:
+            {
+                result.m_type = value_t::number_unsigned;
+                result.m_value = m_lexer.get_number_unsigned();
+                break;
+            }
+
+            case token_type::value_integer:
+            {
+                result.m_type = value_t::number_integer;
+                result.m_value = m_lexer.get_number_integer();
+                break;
+            }
+
+            case token_type::value_float:
+            {
+                result.m_type = value_t::number_float;
+                result.m_value = m_lexer.get_number_float();
+
+                // throw in case of infinity or NAN
+                if (JSON_UNLIKELY(not std::isfinite(result.m_value.number_float)))
+                {
+                    if (allow_exceptions)
+                    {
+                        JSON_THROW(out_of_range::create(406, "number overflow parsing '" +
+                                                        m_lexer.get_token_string() + "'"));
+                    }
+                    expect(token_type::uninitialized);
+                }
+                break;
+            }
+
+            case token_type::parse_error:
+            {
+                // using "uninitialized" to avoid "expected" message
+                if (not expect(token_type::uninitialized))
+                {
+                    return;
+                }
+                break; // LCOV_EXCL_LINE
+            }
+
+            default:
+            {
+                // the last token was unexpected; we expected a value
+                if (not expect(token_type::literal_or_value))
+                {
+                    return;
+                }
+                break; // LCOV_EXCL_LINE
+            }
+        }
+
+        if (keep and callback and not callback(depth, parse_event_t::value, result))
+        {
+            result.m_type = value_t::discarded;
+        }
+    }
+
+    /*!
+    @brief the acutal acceptor
+
+    @invariant 1. The last token is not yet processed. Therefore, the caller
+                  of this function must make sure a token has been read.
+               2. When this function returns, the last token is processed.
+                  That is, the last read character was already considered.
+
+    This invariant makes sure that no token needs to be "unput".
+    */
+    bool accept_internal()
+    {
+        switch (last_token)
+        {
+            case token_type::begin_object:
+            {
+                // read next token
+                get_token();
+
+                // closing } -> we are done
+                if (last_token == token_type::end_object)
+                {
+                    return true;
+                }
+
+                // parse values
+                while (true)
+                {
+                    // parse key
+                    if (last_token != token_type::value_string)
+                    {
+                        return false;
+                    }
+
+                    // parse separator (:)
+                    get_token();
+                    if (last_token != token_type::name_separator)
+                    {
+                        return false;
+                    }
+
+                    // parse value
+                    get_token();
+                    if (not accept_internal())
+                    {
+                        return false;
+                    }
+
+                    // comma -> next value
+                    get_token();
+                    if (last_token == token_type::value_separator)
+                    {
+                        get_token();
+                        continue;
+                    }
+
+                    // closing }
+                    return (last_token == token_type::end_object);
+                }
+            }
+
+            case token_type::begin_array:
+            {
+                // read next token
+                get_token();
+
+                // closing ] -> we are done
+                if (last_token == token_type::end_array)
+                {
+                    return true;
+                }
+
+                // parse values
+                while (true)
+                {
+                    // parse value
+                    if (not accept_internal())
+                    {
+                        return false;
+                    }
+
+                    // comma -> next value
+                    get_token();
+                    if (last_token == token_type::value_separator)
+                    {
+                        get_token();
+                        continue;
+                    }
+
+                    // closing ]
+                    return (last_token == token_type::end_array);
+                }
+            }
+
+            case token_type::value_float:
+            {
+                // reject infinity or NAN
+                return std::isfinite(m_lexer.get_number_float());
+            }
+
+            case token_type::literal_false:
+            case token_type::literal_null:
+            case token_type::literal_true:
+            case token_type::value_integer:
+            case token_type::value_string:
+            case token_type::value_unsigned:
+                return true;
+
+            default: // the last token was unexpected
+                return false;
+        }
+    }
+
+    /// get next token from lexer
+    token_type get_token()
+    {
+        return (last_token = m_lexer.scan());
+    }
+
+    /*!
+    @throw parse_error.101 if expected token did not occur
+    */
+    bool expect(token_type t)
+    {
+        if (JSON_UNLIKELY(t != last_token))
+        {
+            errored = true;
+            expected = t;
+            if (allow_exceptions)
+            {
+                throw_exception();
+            }
+            else
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    [[noreturn]] void throw_exception() const
+    {
+        std::string error_msg = "syntax error - ";
+        if (last_token == token_type::parse_error)
+        {
+            error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" +
+                         m_lexer.get_token_string() + "'";
+        }
+        else
+        {
+            error_msg += "unexpected " + std::string(lexer_t::token_type_name(last_token));
+        }
+
+        if (expected != token_type::uninitialized)
+        {
+            error_msg += "; expected " + std::string(lexer_t::token_type_name(expected));
+        }
+
+        JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg));
+    }
+
+  private:
+    /// current level of recursion
+    int depth = 0;
+    /// callback function
+    const parser_callback_t callback = nullptr;
+    /// the type of the last read token
+    token_type last_token = token_type::uninitialized;
+    /// the lexer
+    lexer_t m_lexer;
+    /// whether a syntax error occurred
+    bool errored = false;
+    /// possible reason for the syntax error
+    token_type expected = token_type::uninitialized;
+    /// whether to throw exceptions in case of errors
+    const bool allow_exceptions = true;
+};
+
+///////////////
+// iterators //
+///////////////
+
+/*!
+@brief an iterator for primitive JSON types
+
+This class models an iterator for primitive JSON types (boolean, number,
+string). It's only purpose is to allow the iterator/const_iterator classes
+to "iterate" over primitive values. Internally, the iterator is modeled by
+a `difference_type` variable. Value begin_value (`0`) models the begin,
+end_value (`1`) models past the end.
+*/
+class primitive_iterator_t
+{
+  public:
+    using difference_type = std::ptrdiff_t;
+
+    constexpr difference_type get_value() const noexcept
+    {
+        return m_it;
+    }
+
+    /// set iterator to a defined beginning
+    void set_begin() noexcept
+    {
+        m_it = begin_value;
+    }
+
+    /// set iterator to a defined past the end
+    void set_end() noexcept
+    {
+        m_it = end_value;
+    }
+
+    /// return whether the iterator can be dereferenced
+    constexpr bool is_begin() const noexcept
+    {
+        return (m_it == begin_value);
+    }
+
+    /// return whether the iterator is at end
+    constexpr bool is_end() const noexcept
+    {
+        return (m_it == end_value);
+    }
+
+    friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return (lhs.m_it == rhs.m_it);
+    }
+
+    friend constexpr bool operator!=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return not(lhs == rhs);
+    }
+
+    friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it < rhs.m_it;
+    }
+
+    friend constexpr bool operator<=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it <= rhs.m_it;
+    }
+
+    friend constexpr bool operator>(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it > rhs.m_it;
+    }
+
+    friend constexpr bool operator>=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it >= rhs.m_it;
+    }
+
+    primitive_iterator_t operator+(difference_type i)
+    {
+        auto result = *this;
+        result += i;
+        return result;
+    }
+
+    friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
+    {
+        return lhs.m_it - rhs.m_it;
+    }
+
+    friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it)
+    {
+        return os << it.m_it;
+    }
+
+    primitive_iterator_t& operator++()
+    {
+        ++m_it;
+        return *this;
+    }
+
+    primitive_iterator_t operator++(int)
+    {
+        auto result = *this;
+        m_it++;
+        return result;
+    }
+
+    primitive_iterator_t& operator--()
+    {
+        --m_it;
+        return *this;
+    }
+
+    primitive_iterator_t operator--(int)
+    {
+        auto result = *this;
+        m_it--;
+        return result;
+    }
+
+    primitive_iterator_t& operator+=(difference_type n)
+    {
+        m_it += n;
+        return *this;
+    }
+
+    primitive_iterator_t& operator-=(difference_type n)
+    {
+        m_it -= n;
+        return *this;
+    }
+
+  private:
+    static constexpr difference_type begin_value = 0;
+    static constexpr difference_type end_value = begin_value + 1;
+
+    /// iterator as signed integer type
+    difference_type m_it = std::numeric_limits<std::ptrdiff_t>::denorm_min();
+};
+
+/*!
+@brief an iterator value
+
+@note This structure could easily be a union, but MSVC currently does not allow
+unions members with complex constructors, see https://github.com/nlohmann/json/pull/105.
+*/
+template<typename BasicJsonType> struct internal_iterator
+{
+    /// iterator for JSON objects
+    typename BasicJsonType::object_t::iterator object_iterator {};
+    /// iterator for JSON arrays
+    typename BasicJsonType::array_t::iterator array_iterator {};
+    /// generic iterator for all other types
+    primitive_iterator_t primitive_iterator {};
+};
+
+template<typename IteratorType> class iteration_proxy;
+
+/*!
+@brief a template for a random access iterator for the @ref basic_json class
+
+This class implements a both iterators (iterator and const_iterator) for the
+@ref basic_json class.
+
+@note An iterator is called *initialized* when a pointer to a JSON value has
+      been set (e.g., by a constructor or a copy assignment). If the iterator is
+      default-constructed, it is *uninitialized* and most methods are undefined.
+      **The library uses assertions to detect calls on uninitialized iterators.**
+
+@requirement The class satisfies the following concept requirements:
+-
+[RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator):
+  The iterator that can be moved to point (forward and backward) to any
+  element in constant time.
+
+@since version 1.0.0, simplified in version 2.0.9
+*/
+template<typename BasicJsonType>
+class iter_impl : public std::iterator<std::random_access_iterator_tag, BasicJsonType>
+{
+    /// allow basic_json to access private members
+    friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>;
+    friend BasicJsonType;
+    friend iteration_proxy<iter_impl>;
+
+    using object_t = typename BasicJsonType::object_t;
+    using array_t = typename BasicJsonType::array_t;
+    // make sure BasicJsonType is basic_json or const basic_json
+    static_assert(is_basic_json<typename std::remove_const<BasicJsonType>::type>::value,
+                  "iter_impl only accepts (const) basic_json");
+
+  public:
+    /// the type of the values when the iterator is dereferenced
+    using value_type = typename BasicJsonType::value_type;
+    /// a type to represent differences between iterators
+    using difference_type = typename BasicJsonType::difference_type;
+    /// defines a pointer to the type iterated over (value_type)
+    using pointer = typename std::conditional<std::is_const<BasicJsonType>::value,
+          typename BasicJsonType::const_pointer,
+          typename BasicJsonType::pointer>::type;
+    /// defines a reference to the type iterated over (value_type)
+    using reference =
+        typename std::conditional<std::is_const<BasicJsonType>::value,
+        typename BasicJsonType::const_reference,
+        typename BasicJsonType::reference>::type;
+    /// the category of the iterator
+    using iterator_category = std::bidirectional_iterator_tag;
+
+    /// default constructor
+    iter_impl() = default;
+
+    /*!
+    @brief constructor for a given JSON instance
+    @param[in] object  pointer to a JSON object for this iterator
+    @pre object != nullptr
+    @post The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    explicit iter_impl(pointer object) noexcept : m_object(object)
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                m_it.object_iterator = typename object_t::iterator();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_it.array_iterator = typename array_t::iterator();
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator = primitive_iterator_t();
+                break;
+            }
+        }
+    }
+
+    /*!
+    @note The conventional copy constructor and copy assignment are implicitly
+          defined. Combined with the following converting constructor and
+          assignment, they support: (1) copy from iterator to iterator, (2)
+          copy from const iterator to const iterator, and (3) conversion from
+          iterator to const iterator. However conversion from const iterator
+          to iterator is not defined.
+    */
+
+    /*!
+    @brief converting constructor
+    @param[in] other  non-const iterator to copy from
+    @note It is not checked whether @a other is initialized.
+    */
+    iter_impl(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept
+        : m_object(other.m_object), m_it(other.m_it) {}
+
+    /*!
+    @brief converting assignment
+    @param[in,out] other  non-const iterator to copy from
+    @return const/non-const iterator
+    @note It is not checked whether @a other is initialized.
+    */
+    iter_impl& operator=(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept
+    {
+        m_object = other.m_object;
+        m_it = other.m_it;
+        return *this;
+    }
+
+  private:
+    /*!
+    @brief set the iterator to the first value
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    void set_begin() noexcept
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                m_it.object_iterator = m_object->m_value.object->begin();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_it.array_iterator = m_object->m_value.array->begin();
+                break;
+            }
+
+            case value_t::null:
+            {
+                // set to end so begin()==end() is true: null is empty
+                m_it.primitive_iterator.set_end();
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator.set_begin();
+                break;
+            }
+        }
+    }
+
+    /*!
+    @brief set the iterator past the last value
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    void set_end() noexcept
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                m_it.object_iterator = m_object->m_value.object->end();
+                break;
+            }
+
+            case value_t::array:
+            {
+                m_it.array_iterator = m_object->m_value.array->end();
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator.set_end();
+                break;
+            }
+        }
+    }
+
+  public:
+    /*!
+    @brief return a reference to the value pointed to by the iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    reference operator*() const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                assert(m_it.object_iterator != m_object->m_value.object->end());
+                return m_it.object_iterator->second;
+            }
+
+            case value_t::array:
+            {
+                assert(m_it.array_iterator != m_object->m_value.array->end());
+                return *m_it.array_iterator;
+            }
+
+            case value_t::null:
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+
+            default:
+            {
+                if (JSON_LIKELY(m_it.primitive_iterator.is_begin()))
+                {
+                    return *m_object;
+                }
+
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+            }
+        }
+    }
+
+    /*!
+    @brief dereference the iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    pointer operator->() const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                assert(m_it.object_iterator != m_object->m_value.object->end());
+                return &(m_it.object_iterator->second);
+            }
+
+            case value_t::array:
+            {
+                assert(m_it.array_iterator != m_object->m_value.array->end());
+                return &*m_it.array_iterator;
+            }
+
+            default:
+            {
+                if (JSON_LIKELY(m_it.primitive_iterator.is_begin()))
+                {
+                    return m_object;
+                }
+
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+            }
+        }
+    }
+
+    /*!
+    @brief post-increment (it++)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator++(int)
+    {
+        auto result = *this;
+        ++(*this);
+        return result;
+    }
+
+    /*!
+    @brief pre-increment (++it)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator++()
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                std::advance(m_it.object_iterator, 1);
+                break;
+            }
+
+            case value_t::array:
+            {
+                std::advance(m_it.array_iterator, 1);
+                break;
+            }
+
+            default:
+            {
+                ++m_it.primitive_iterator;
+                break;
+            }
+        }
+
+        return *this;
+    }
+
+    /*!
+    @brief post-decrement (it--)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator--(int)
+    {
+        auto result = *this;
+        --(*this);
+        return result;
+    }
+
+    /*!
+    @brief pre-decrement (--it)
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator--()
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+            {
+                std::advance(m_it.object_iterator, -1);
+                break;
+            }
+
+            case value_t::array:
+            {
+                std::advance(m_it.array_iterator, -1);
+                break;
+            }
+
+            default:
+            {
+                --m_it.primitive_iterator;
+                break;
+            }
+        }
+
+        return *this;
+    }
+
+    /*!
+    @brief  comparison: equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator==(const iter_impl& other) const
+    {
+        // if objects are not the same, the comparison is undefined
+        if (JSON_UNLIKELY(m_object != other.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
+        }
+
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                return (m_it.object_iterator == other.m_it.object_iterator);
+
+            case value_t::array:
+                return (m_it.array_iterator == other.m_it.array_iterator);
+
+            default:
+                return (m_it.primitive_iterator == other.m_it.primitive_iterator);
+        }
+    }
+
+    /*!
+    @brief  comparison: not equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator!=(const iter_impl& other) const
+    {
+        return not operator==(other);
+    }
+
+    /*!
+    @brief  comparison: smaller
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator<(const iter_impl& other) const
+    {
+        // if objects are not the same, the comparison is undefined
+        if (JSON_UNLIKELY(m_object != other.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
+        }
+
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators"));
+
+            case value_t::array:
+                return (m_it.array_iterator < other.m_it.array_iterator);
+
+            default:
+                return (m_it.primitive_iterator < other.m_it.primitive_iterator);
+        }
+    }
+
+    /*!
+    @brief  comparison: less than or equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator<=(const iter_impl& other) const
+    {
+        return not other.operator < (*this);
+    }
+
+    /*!
+    @brief  comparison: greater than
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator>(const iter_impl& other) const
+    {
+        return not operator<=(other);
+    }
+
+    /*!
+    @brief  comparison: greater than or equal
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    bool operator>=(const iter_impl& other) const
+    {
+        return not operator<(other);
+    }
+
+    /*!
+    @brief  add to iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator+=(difference_type i)
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
+
+            case value_t::array:
+            {
+                std::advance(m_it.array_iterator, i);
+                break;
+            }
+
+            default:
+            {
+                m_it.primitive_iterator += i;
+                break;
+            }
+        }
+
+        return *this;
+    }
+
+    /*!
+    @brief  subtract from iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl& operator-=(difference_type i)
+    {
+        return operator+=(-i);
+    }
+
+    /*!
+    @brief  add to iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator+(difference_type i) const
+    {
+        auto result = *this;
+        result += i;
+        return result;
+    }
+
+    /*!
+    @brief  addition of distance and iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    friend iter_impl operator+(difference_type i, const iter_impl& it)
+    {
+        auto result = it;
+        result += i;
+        return result;
+    }
+
+    /*!
+    @brief  subtract from iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    iter_impl operator-(difference_type i) const
+    {
+        auto result = *this;
+        result -= i;
+        return result;
+    }
+
+    /*!
+    @brief  return difference
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    difference_type operator-(const iter_impl& other) const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
+
+            case value_t::array:
+                return m_it.array_iterator - other.m_it.array_iterator;
+
+            default:
+                return m_it.primitive_iterator - other.m_it.primitive_iterator;
+        }
+    }
+
+    /*!
+    @brief  access to successor
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    reference operator[](difference_type n) const
+    {
+        assert(m_object != nullptr);
+
+        switch (m_object->m_type)
+        {
+            case value_t::object:
+                JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators"));
+
+            case value_t::array:
+                return *std::next(m_it.array_iterator, n);
+
+            case value_t::null:
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+
+            default:
+            {
+                if (JSON_LIKELY(m_it.primitive_iterator.get_value() == -n))
+                {
+                    return *m_object;
+                }
+
+                JSON_THROW(invalid_iterator::create(214, "cannot get value"));
+            }
+        }
+    }
+
+    /*!
+    @brief  return the key of an object iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    typename object_t::key_type key() const
+    {
+        assert(m_object != nullptr);
+
+        if (JSON_LIKELY(m_object->is_object()))
+        {
+            return m_it.object_iterator->first;
+        }
+
+        JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators"));
+    }
+
+    /*!
+    @brief  return the value of an iterator
+    @pre The iterator is initialized; i.e. `m_object != nullptr`.
+    */
+    reference value() const
+    {
+        return operator*();
+    }
+
+  private:
+    /// associated JSON instance
+    pointer m_object = nullptr;
+    /// the actual iterator of the associated instance
+    internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it = {};
+};
+
+/// proxy class for the iterator_wrapper functions
+template<typename IteratorType> class iteration_proxy
+{
+  private:
+    /// helper class for iteration
+    class iteration_proxy_internal
+    {
+      private:
+        /// the iterator
+        IteratorType anchor;
+        /// an index for arrays (used to create key names)
+        std::size_t array_index = 0;
+
+      public:
+        explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {}
+
+        /// dereference operator (needed for range-based for)
+        iteration_proxy_internal& operator*()
+        {
+            return *this;
+        }
+
+        /// increment operator (needed for range-based for)
+        iteration_proxy_internal& operator++()
+        {
+            ++anchor;
+            ++array_index;
+
+            return *this;
+        }
+
+        /// inequality operator (needed for range-based for)
+        bool operator!=(const iteration_proxy_internal& o) const noexcept
+        {
+            return anchor != o.anchor;
+        }
+
+        /// return key of the iterator
+        std::string key() const
+        {
+            assert(anchor.m_object != nullptr);
+
+            switch (anchor.m_object->type())
+            {
+                // use integer array index as key
+                case value_t::array:
+                    return std::to_string(array_index);
+
+                // use key from the object
+                case value_t::object:
+                    return anchor.key();
+
+                // use an empty key for all primitive types
+                default:
+                    return "";
+            }
+        }
+
+        /// return value of the iterator
+        typename IteratorType::reference value() const
+        {
+            return anchor.value();
+        }
+    };
+
+    /// the container to iterate
+    typename IteratorType::reference container;
+
+  public:
+    /// construct iteration proxy from a container
+    explicit iteration_proxy(typename IteratorType::reference cont)
+        : container(cont) {}
+
+    /// return iterator begin (needed for range-based for)
+    iteration_proxy_internal begin() noexcept
+    {
+        return iteration_proxy_internal(container.begin());
+    }
+
+    /// return iterator end (needed for range-based for)
+    iteration_proxy_internal end() noexcept
+    {
+        return iteration_proxy_internal(container.end());
+    }
+};
+
+/*!
+@brief a template for a reverse iterator class
+
+@tparam Base the base iterator type to reverse. Valid types are @ref
+iterator (to create @ref reverse_iterator) and @ref const_iterator (to
+create @ref const_reverse_iterator).
+
+@requirement The class satisfies the following concept requirements:
+-
+[RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator):
+  The iterator that can be moved to point (forward and backward) to any
+  element in constant time.
+- [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator):
+  It is possible to write to the pointed-to element (only if @a Base is
+  @ref iterator).
+
+@since version 1.0.0
+*/
+template<typename Base>
+class json_reverse_iterator : public std::reverse_iterator<Base>
+{
+  public:
+    using difference_type = std::ptrdiff_t;
+    /// shortcut to the reverse iterator adaptor
+    using base_iterator = std::reverse_iterator<Base>;
+    /// the reference type for the pointed-to element
+    using reference = typename Base::reference;
+
+    /// create reverse iterator from iterator
+    json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
+        : base_iterator(it) {}
+
+    /// create reverse iterator from base class
+    json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {}
+
+    /// post-increment (it++)
+    json_reverse_iterator operator++(int)
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator++(1));
+    }
+
+    /// pre-increment (++it)
+    json_reverse_iterator& operator++()
+    {
+        return static_cast<json_reverse_iterator&>(base_iterator::operator++());
+    }
+
+    /// post-decrement (it--)
+    json_reverse_iterator operator--(int)
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator--(1));
+    }
+
+    /// pre-decrement (--it)
+    json_reverse_iterator& operator--()
+    {
+        return static_cast<json_reverse_iterator&>(base_iterator::operator--());
+    }
+
+    /// add to iterator
+    json_reverse_iterator& operator+=(difference_type i)
+    {
+        return static_cast<json_reverse_iterator&>(base_iterator::operator+=(i));
+    }
+
+    /// add to iterator
+    json_reverse_iterator operator+(difference_type i) const
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator+(i));
+    }
+
+    /// subtract from iterator
+    json_reverse_iterator operator-(difference_type i) const
+    {
+        return static_cast<json_reverse_iterator>(base_iterator::operator-(i));
+    }
+
+    /// return difference
+    difference_type operator-(const json_reverse_iterator& other) const
+    {
+        return base_iterator(*this) - base_iterator(other);
+    }
+
+    /// access to successor
+    reference operator[](difference_type n) const
+    {
+        return *(this->operator+(n));
+    }
+
+    /// return the key of an object iterator
+    auto key() const -> decltype(std::declval<Base>().key())
+    {
+        auto it = --this->base();
+        return it.key();
+    }
+
+    /// return the value of an iterator
+    reference value() const
+    {
+        auto it = --this->base();
+        return it.operator * ();
+    }
+};
+
+/////////////////////
+// output adapters //
+/////////////////////
+
+/// abstract output adapter interface
+template<typename CharType> struct output_adapter_protocol
+{
+    virtual void write_character(CharType c) = 0;
+    virtual void write_characters(const CharType* s, std::size_t length) = 0;
+    virtual ~output_adapter_protocol() = default;
+};
+
+/// a type to simplify interfaces
+template<typename CharType>
+using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>;
+
+/// output adapter for byte vectors
+template<typename CharType>
+class output_vector_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_vector_adapter(std::vector<CharType>& vec) : v(vec) {}
+
+    void write_character(CharType c) override
+    {
+        v.push_back(c);
+    }
+
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        std::copy(s, s + length, std::back_inserter(v));
+    }
+
+  private:
+    std::vector<CharType>& v;
+};
+
+/// output adapter for output streams
+template<typename CharType>
+class output_stream_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_stream_adapter(std::basic_ostream<CharType>& s) : stream(s) {}
+
+    void write_character(CharType c) override
+    {
+        stream.put(c);
+    }
+
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        stream.write(s, static_cast<std::streamsize>(length));
+    }
+
+  private:
+    std::basic_ostream<CharType>& stream;
+};
+
+/// output adapter for basic_string
+template<typename CharType>
+class output_string_adapter : public output_adapter_protocol<CharType>
+{
+  public:
+    explicit output_string_adapter(std::basic_string<CharType>& s) : str(s) {}
+
+    void write_character(CharType c) override
+    {
+        str.push_back(c);
+    }
+
+    void write_characters(const CharType* s, std::size_t length) override
+    {
+        str.append(s, length);
+    }
+
+  private:
+    std::basic_string<CharType>& str;
+};
+
+template<typename CharType>
+class output_adapter
+{
+  public:
+    output_adapter(std::vector<CharType>& vec)
+        : oa(std::make_shared<output_vector_adapter<CharType>>(vec)) {}
+
+    output_adapter(std::basic_ostream<CharType>& s)
+        : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {}
+
+    output_adapter(std::basic_string<CharType>& s)
+        : oa(std::make_shared<output_string_adapter<CharType>>(s)) {}
+
+    operator output_adapter_t<CharType>()
+    {
+        return oa;
+    }
+
+  private:
+    output_adapter_t<CharType> oa = nullptr;
+};
+
+//////////////////////////////
+// binary reader and writer //
+//////////////////////////////
+
+/*!
+@brief deserialization of CBOR and MessagePack values
+*/
+template<typename BasicJsonType>
+class binary_reader
+{
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+
+  public:
+    /*!
+    @brief create a binary reader
+
+    @param[in] adapter  input adapter to read from
+    */
+    explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter))
+    {
+        assert(ia);
+    }
+
+    /*!
+    @brief create a JSON value from CBOR input
+
+    @param[in] strict  whether to expect the input to be consumed completed
+    @return JSON value created from CBOR input
+
+    @throw parse_error.110 if input ended unexpectedly or the end of file was
+                           not reached when @a strict was set to true
+    @throw parse_error.112 if unsupported byte was read
+    */
+    BasicJsonType parse_cbor(const bool strict)
+    {
+        const auto res = parse_cbor_internal();
+        if (strict)
+        {
+            get();
+            check_eof(true);
+        }
+        return res;
+    }
+
+    /*!
+    @brief create a JSON value from MessagePack input
+
+    @param[in] strict  whether to expect the input to be consumed completed
+    @return JSON value created from MessagePack input
+
+    @throw parse_error.110 if input ended unexpectedly or the end of file was
+                           not reached when @a strict was set to true
+    @throw parse_error.112 if unsupported byte was read
+    */
+    BasicJsonType parse_msgpack(const bool strict)
+    {
+        const auto res = parse_msgpack_internal();
+        if (strict)
+        {
+            get();
+            check_eof(true);
+        }
+        return res;
+    }
+
+    /*!
+    @brief determine system byte order
+
+    @return true if and only if system's byte order is little endian
+
+    @note from http://stackoverflow.com/a/1001328/266378
+    */
+    static constexpr bool little_endianess(int num = 1) noexcept
+    {
+        return (*reinterpret_cast<char*>(&num) == 1);
+    }
+
+  private:
+    /*!
+    @param[in] get_char  whether a new character should be retrieved from the
+                         input (true, default) or whether the last read
+                         character should be considered instead
+    */
+    BasicJsonType parse_cbor_internal(const bool get_char = true)
+    {
+        switch (get_char ? get() : current)
+        {
+            // EOF
+            case std::char_traits<char>::eof():
+                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+
+            // Integer 0x00..0x17 (0..23)
+            case 0x00:
+            case 0x01:
+            case 0x02:
+            case 0x03:
+            case 0x04:
+            case 0x05:
+            case 0x06:
+            case 0x07:
+            case 0x08:
+            case 0x09:
+            case 0x0a:
+            case 0x0b:
+            case 0x0c:
+            case 0x0d:
+            case 0x0e:
+            case 0x0f:
+            case 0x10:
+            case 0x11:
+            case 0x12:
+            case 0x13:
+            case 0x14:
+            case 0x15:
+            case 0x16:
+            case 0x17:
+                return static_cast<number_unsigned_t>(current);
+
+            case 0x18: // Unsigned integer (one-byte uint8_t follows)
+                return get_number<uint8_t>();
+
+            case 0x19: // Unsigned integer (two-byte uint16_t follows)
+                return get_number<uint16_t>();
+
+            case 0x1a: // Unsigned integer (four-byte uint32_t follows)
+                return get_number<uint32_t>();
+
+            case 0x1b: // Unsigned integer (eight-byte uint64_t follows)
+                return get_number<uint64_t>();
+
+            // Negative integer -1-0x00..-1-0x17 (-1..-24)
+            case 0x20:
+            case 0x21:
+            case 0x22:
+            case 0x23:
+            case 0x24:
+            case 0x25:
+            case 0x26:
+            case 0x27:
+            case 0x28:
+            case 0x29:
+            case 0x2a:
+            case 0x2b:
+            case 0x2c:
+            case 0x2d:
+            case 0x2e:
+            case 0x2f:
+            case 0x30:
+            case 0x31:
+            case 0x32:
+            case 0x33:
+            case 0x34:
+            case 0x35:
+            case 0x36:
+            case 0x37:
+                return static_cast<int8_t>(0x20 - 1 - current);
+
+            case 0x38: // Negative integer (one-byte uint8_t follows)
+            {
+                // must be uint8_t !
+                return static_cast<number_integer_t>(-1) - get_number<uint8_t>();
+            }
+
+            case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
+            {
+                return static_cast<number_integer_t>(-1) - get_number<uint16_t>();
+            }
+
+            case 0x3a: // Negative integer -1-n (four-byte uint32_t follows)
+            {
+                return static_cast<number_integer_t>(-1) - get_number<uint32_t>();
+            }
+
+            case 0x3b: // Negative integer -1-n (eight-byte uint64_t follows)
+            {
+                return static_cast<number_integer_t>(-1) -
+                       static_cast<number_integer_t>(get_number<uint64_t>());
+            }
+
+            // UTF-8 string (0x00..0x17 bytes follow)
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6a:
+            case 0x6b:
+            case 0x6c:
+            case 0x6d:
+            case 0x6e:
+            case 0x6f:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+            case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
+            case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
+            case 0x7f: // UTF-8 string (indefinite length)
+            {
+                return get_cbor_string();
+            }
+
+            // array (0x00..0x17 data items follow)
+            case 0x80:
+            case 0x81:
+            case 0x82:
+            case 0x83:
+            case 0x84:
+            case 0x85:
+            case 0x86:
+            case 0x87:
+            case 0x88:
+            case 0x89:
+            case 0x8a:
+            case 0x8b:
+            case 0x8c:
+            case 0x8d:
+            case 0x8e:
+            case 0x8f:
+            case 0x90:
+            case 0x91:
+            case 0x92:
+            case 0x93:
+            case 0x94:
+            case 0x95:
+            case 0x96:
+            case 0x97:
+            {
+                return get_cbor_array(current & 0x1f);
+            }
+
+            case 0x98: // array (one-byte uint8_t for n follows)
+            {
+                return get_cbor_array(get_number<uint8_t>());
+            }
+
+            case 0x99: // array (two-byte uint16_t for n follow)
+            {
+                return get_cbor_array(get_number<uint16_t>());
+            }
+
+            case 0x9a: // array (four-byte uint32_t for n follow)
+            {
+                return get_cbor_array(get_number<uint32_t>());
+            }
+
+            case 0x9b: // array (eight-byte uint64_t for n follow)
+            {
+                return get_cbor_array(get_number<uint64_t>());
+            }
+
+            case 0x9f: // array (indefinite length)
+            {
+                BasicJsonType result = value_t::array;
+                while (get() != 0xff)
+                {
+                    result.push_back(parse_cbor_internal(false));
+                }
+                return result;
+            }
+
+            // map (0x00..0x17 pairs of data items follow)
+            case 0xa0:
+            case 0xa1:
+            case 0xa2:
+            case 0xa3:
+            case 0xa4:
+            case 0xa5:
+            case 0xa6:
+            case 0xa7:
+            case 0xa8:
+            case 0xa9:
+            case 0xaa:
+            case 0xab:
+            case 0xac:
+            case 0xad:
+            case 0xae:
+            case 0xaf:
+            case 0xb0:
+            case 0xb1:
+            case 0xb2:
+            case 0xb3:
+            case 0xb4:
+            case 0xb5:
+            case 0xb6:
+            case 0xb7:
+            {
+                return get_cbor_object(current & 0x1f);
+            }
+
+            case 0xb8: // map (one-byte uint8_t for n follows)
+            {
+                return get_cbor_object(get_number<uint8_t>());
+            }
+
+            case 0xb9: // map (two-byte uint16_t for n follow)
+            {
+                return get_cbor_object(get_number<uint16_t>());
+            }
+
+            case 0xba: // map (four-byte uint32_t for n follow)
+            {
+                return get_cbor_object(get_number<uint32_t>());
+            }
+
+            case 0xbb: // map (eight-byte uint64_t for n follow)
+            {
+                return get_cbor_object(get_number<uint64_t>());
+            }
+
+            case 0xbf: // map (indefinite length)
+            {
+                BasicJsonType result = value_t::object;
+                while (get() != 0xff)
+                {
+                    auto key = get_cbor_string();
+                    result[key] = parse_cbor_internal();
+                }
+                return result;
+            }
+
+            case 0xf4: // false
+            {
+                return false;
+            }
+
+            case 0xf5: // true
+            {
+                return true;
+            }
+
+            case 0xf6: // null
+            {
+                return value_t::null;
+            }
+
+            case 0xf9: // Half-Precision Float (two-byte IEEE 754)
+            {
+                const int byte1 = get();
+                check_eof();
+                const int byte2 = get();
+                check_eof();
+
+                // code from RFC 7049, Appendix D, Figure 3:
+                // As half-precision floating-point numbers were only added
+                // to IEEE 754 in 2008, today's programming platforms often
+                // still only have limited support for them. It is very
+                // easy to include at least decoding support for them even
+                // without such support. An example of a small decoder for
+                // half-precision floating-point numbers in the C language
+                // is shown in Fig. 3.
+                const int half = (byte1 << 8) + byte2;
+                const int exp = (half >> 10) & 0x1f;
+                const int mant = half & 0x3ff;
+                double val;
+                if (exp == 0)
+                {
+                    val = std::ldexp(mant, -24);
+                }
+                else if (exp != 31)
+                {
+                    val = std::ldexp(mant + 1024, exp - 25);
+                }
+                else
+                {
+                    val = (mant == 0) ? std::numeric_limits<double>::infinity()
+                          : std::numeric_limits<double>::quiet_NaN();
+                }
+                return (half & 0x8000) != 0 ? -val : val;
+            }
+
+            case 0xfa: // Single-Precision Float (four-byte IEEE 754)
+            {
+                return get_number<float>();
+            }
+
+            case 0xfb: // Double-Precision Float (eight-byte IEEE 754)
+            {
+                return get_number<double>();
+            }
+
+            default: // anything else (0xFF is handled inside the other types)
+            {
+                std::stringstream ss;
+                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str()));
+            }
+        }
+    }
+
+    BasicJsonType parse_msgpack_internal()
+    {
+        switch (get())
+        {
+            // EOF
+            case std::char_traits<char>::eof():
+                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+
+            // positive fixint
+            case 0x00:
+            case 0x01:
+            case 0x02:
+            case 0x03:
+            case 0x04:
+            case 0x05:
+            case 0x06:
+            case 0x07:
+            case 0x08:
+            case 0x09:
+            case 0x0a:
+            case 0x0b:
+            case 0x0c:
+            case 0x0d:
+            case 0x0e:
+            case 0x0f:
+            case 0x10:
+            case 0x11:
+            case 0x12:
+            case 0x13:
+            case 0x14:
+            case 0x15:
+            case 0x16:
+            case 0x17:
+            case 0x18:
+            case 0x19:
+            case 0x1a:
+            case 0x1b:
+            case 0x1c:
+            case 0x1d:
+            case 0x1e:
+            case 0x1f:
+            case 0x20:
+            case 0x21:
+            case 0x22:
+            case 0x23:
+            case 0x24:
+            case 0x25:
+            case 0x26:
+            case 0x27:
+            case 0x28:
+            case 0x29:
+            case 0x2a:
+            case 0x2b:
+            case 0x2c:
+            case 0x2d:
+            case 0x2e:
+            case 0x2f:
+            case 0x30:
+            case 0x31:
+            case 0x32:
+            case 0x33:
+            case 0x34:
+            case 0x35:
+            case 0x36:
+            case 0x37:
+            case 0x38:
+            case 0x39:
+            case 0x3a:
+            case 0x3b:
+            case 0x3c:
+            case 0x3d:
+            case 0x3e:
+            case 0x3f:
+            case 0x40:
+            case 0x41:
+            case 0x42:
+            case 0x43:
+            case 0x44:
+            case 0x45:
+            case 0x46:
+            case 0x47:
+            case 0x48:
+            case 0x49:
+            case 0x4a:
+            case 0x4b:
+            case 0x4c:
+            case 0x4d:
+            case 0x4e:
+            case 0x4f:
+            case 0x50:
+            case 0x51:
+            case 0x52:
+            case 0x53:
+            case 0x54:
+            case 0x55:
+            case 0x56:
+            case 0x57:
+            case 0x58:
+            case 0x59:
+            case 0x5a:
+            case 0x5b:
+            case 0x5c:
+            case 0x5d:
+            case 0x5e:
+            case 0x5f:
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6a:
+            case 0x6b:
+            case 0x6c:
+            case 0x6d:
+            case 0x6e:
+            case 0x6f:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            case 0x78:
+            case 0x79:
+            case 0x7a:
+            case 0x7b:
+            case 0x7c:
+            case 0x7d:
+            case 0x7e:
+            case 0x7f:
+                return static_cast<number_unsigned_t>(current);
+
+            // fixmap
+            case 0x80:
+            case 0x81:
+            case 0x82:
+            case 0x83:
+            case 0x84:
+            case 0x85:
+            case 0x86:
+            case 0x87:
+            case 0x88:
+            case 0x89:
+            case 0x8a:
+            case 0x8b:
+            case 0x8c:
+            case 0x8d:
+            case 0x8e:
+            case 0x8f:
+            {
+                return get_msgpack_object(current & 0x0f);
+            }
+
+            // fixarray
+            case 0x90:
+            case 0x91:
+            case 0x92:
+            case 0x93:
+            case 0x94:
+            case 0x95:
+            case 0x96:
+            case 0x97:
+            case 0x98:
+            case 0x99:
+            case 0x9a:
+            case 0x9b:
+            case 0x9c:
+            case 0x9d:
+            case 0x9e:
+            case 0x9f:
+            {
+                return get_msgpack_array(current & 0x0f);
+            }
+
+            // fixstr
+            case 0xa0:
+            case 0xa1:
+            case 0xa2:
+            case 0xa3:
+            case 0xa4:
+            case 0xa5:
+            case 0xa6:
+            case 0xa7:
+            case 0xa8:
+            case 0xa9:
+            case 0xaa:
+            case 0xab:
+            case 0xac:
+            case 0xad:
+            case 0xae:
+            case 0xaf:
+            case 0xb0:
+            case 0xb1:
+            case 0xb2:
+            case 0xb3:
+            case 0xb4:
+            case 0xb5:
+            case 0xb6:
+            case 0xb7:
+            case 0xb8:
+            case 0xb9:
+            case 0xba:
+            case 0xbb:
+            case 0xbc:
+            case 0xbd:
+            case 0xbe:
+            case 0xbf:
+                return get_msgpack_string();
+
+            case 0xc0: // nil
+                return value_t::null;
+
+            case 0xc2: // false
+                return false;
+
+            case 0xc3: // true
+                return true;
+
+            case 0xca: // float 32
+                return get_number<float>();
+
+            case 0xcb: // float 64
+                return get_number<double>();
+
+            case 0xcc: // uint 8
+                return get_number<uint8_t>();
+
+            case 0xcd: // uint 16
+                return get_number<uint16_t>();
+
+            case 0xce: // uint 32
+                return get_number<uint32_t>();
+
+            case 0xcf: // uint 64
+                return get_number<uint64_t>();
+
+            case 0xd0: // int 8
+                return get_number<int8_t>();
+
+            case 0xd1: // int 16
+                return get_number<int16_t>();
+
+            case 0xd2: // int 32
+                return get_number<int32_t>();
+
+            case 0xd3: // int 64
+                return get_number<int64_t>();
+
+            case 0xd9: // str 8
+            case 0xda: // str 16
+            case 0xdb: // str 32
+                return get_msgpack_string();
+
+            case 0xdc: // array 16
+            {
+                return get_msgpack_array(get_number<uint16_t>());
+            }
+
+            case 0xdd: // array 32
+            {
+                return get_msgpack_array(get_number<uint32_t>());
+            }
+
+            case 0xde: // map 16
+            {
+                return get_msgpack_object(get_number<uint16_t>());
+            }
+
+            case 0xdf: // map 32
+            {
+                return get_msgpack_object(get_number<uint32_t>());
+            }
+
+            // positive fixint
+            case 0xe0:
+            case 0xe1:
+            case 0xe2:
+            case 0xe3:
+            case 0xe4:
+            case 0xe5:
+            case 0xe6:
+            case 0xe7:
+            case 0xe8:
+            case 0xe9:
+            case 0xea:
+            case 0xeb:
+            case 0xec:
+            case 0xed:
+            case 0xee:
+            case 0xef:
+            case 0xf0:
+            case 0xf1:
+            case 0xf2:
+            case 0xf3:
+            case 0xf4:
+            case 0xf5:
+            case 0xf6:
+            case 0xf7:
+            case 0xf8:
+            case 0xf9:
+            case 0xfa:
+            case 0xfb:
+            case 0xfc:
+            case 0xfd:
+            case 0xfe:
+            case 0xff:
+                return static_cast<int8_t>(current);
+
+            default: // anything else
+            {
+                std::stringstream ss;
+                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                JSON_THROW(parse_error::create(112, chars_read,
+                                               "error reading MessagePack; last byte: 0x" + ss.str()));
+            }
+        }
+    }
+
+    /*!
+    @brief get next character from the input
+
+    This function provides the interface to the used input adapter. It does
+    not throw in case the input reached EOF, but returns
+    `std::char_traits<char>::eof()` in that case.
+
+    @return character read from the input
+    */
+    int get()
+    {
+        ++chars_read;
+        return (current = ia->get_character());
+    }
+
+    /*
+    @brief read a number from the input
+
+    @tparam NumberType the type of the number
+
+    @return number of type @a NumberType
+
+    @note This function needs to respect the system's endianess, because
+          bytes in CBOR and MessagePack are stored in network order (big
+          endian) and therefore need reordering on little endian systems.
+
+    @throw parse_error.110 if input has less than `sizeof(NumberType)` bytes
+    */
+    template<typename NumberType> NumberType get_number()
+    {
+        // step 1: read input into array with system's byte order
+        std::array<uint8_t, sizeof(NumberType)> vec;
+        for (std::size_t i = 0; i < sizeof(NumberType); ++i)
+        {
+            get();
+            check_eof();
+
+            // reverse byte order prior to conversion if necessary
+            if (is_little_endian)
+            {
+                vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current);
+            }
+            else
+            {
+                vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE
+            }
+        }
+
+        // step 2: convert array into number of type T and return
+        NumberType result;
+        std::memcpy(&result, vec.data(), sizeof(NumberType));
+        return result;
+    }
+
+    /*!
+    @brief create a string by reading characters from the input
+
+    @param[in] len number of bytes to read
+
+    @note We can not reserve @a len bytes for the result, because @a len
+          may be too large. Usually, @ref check_eof() detects the end of
+          the input before we run out of string memory.
+
+    @return string created by reading @a len bytes
+
+    @throw parse_error.110 if input has less than @a len bytes
+    */
+    template<typename NumberType>
+    std::string get_string(const NumberType len)
+    {
+        std::string result;
+        std::generate_n(std::back_inserter(result), len, [this]()
+        {
+            get();
+            check_eof();
+            return current;
+        });
+        return result;
+    }
+
+    /*!
+    @brief reads a CBOR string
+
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
+    Additionally, CBOR's strings with indefinite lengths are supported.
+
+    @return string
+
+    @throw parse_error.110 if input ended
+    @throw parse_error.113 if an unexpected byte is read
+    */
+    std::string get_cbor_string()
+    {
+        check_eof();
+
+        switch (current)
+        {
+            // UTF-8 string (0x00..0x17 bytes follow)
+            case 0x60:
+            case 0x61:
+            case 0x62:
+            case 0x63:
+            case 0x64:
+            case 0x65:
+            case 0x66:
+            case 0x67:
+            case 0x68:
+            case 0x69:
+            case 0x6a:
+            case 0x6b:
+            case 0x6c:
+            case 0x6d:
+            case 0x6e:
+            case 0x6f:
+            case 0x70:
+            case 0x71:
+            case 0x72:
+            case 0x73:
+            case 0x74:
+            case 0x75:
+            case 0x76:
+            case 0x77:
+            {
+                return get_string(current & 0x1f);
+            }
+
+            case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
+            {
+                return get_string(get_number<uint8_t>());
+            }
+
+            case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
+            {
+                return get_string(get_number<uint16_t>());
+            }
+
+            case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
+            {
+                return get_string(get_number<uint32_t>());
+            }
+
+            case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
+            {
+                return get_string(get_number<uint64_t>());
+            }
+
+            case 0x7f: // UTF-8 string (indefinite length)
+            {
+                std::string result;
+                while (get() != 0xff)
+                {
+                    check_eof();
+                    result.push_back(static_cast<char>(current));
+                }
+                return result;
+            }
+
+            default:
+            {
+                std::stringstream ss;
+                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str()));
+            }
+        }
+    }
+
+    template<typename NumberType>
+    BasicJsonType get_cbor_array(const NumberType len)
+    {
+        BasicJsonType result = value_t::array;
+        std::generate_n(std::back_inserter(*result.m_value.array), len, [this]()
+        {
+            return parse_cbor_internal();
+        });
+        return result;
+    }
+
+    template<typename NumberType>
+    BasicJsonType get_cbor_object(const NumberType len)
+    {
+        BasicJsonType result = value_t::object;
+        std::generate_n(std::inserter(*result.m_value.object,
+                                      result.m_value.object->end()),
+                        len, [this]()
+        {
+            get();
+            auto key = get_cbor_string();
+            auto val = parse_cbor_internal();
+            return std::make_pair(std::move(key), std::move(val));
+        });
+        return result;
+    }
+
+    /*!
+    @brief reads a MessagePack string
+
+    This function first reads starting bytes to determine the expected
+    string length and then copies this number of bytes into a string.
+
+    @return string
+
+    @throw parse_error.110 if input ended
+    @throw parse_error.113 if an unexpected byte is read
+    */
+    std::string get_msgpack_string()
+    {
+        check_eof();
+
+        switch (current)
+        {
+            // fixstr
+            case 0xa0:
+            case 0xa1:
+            case 0xa2:
+            case 0xa3:
+            case 0xa4:
+            case 0xa5:
+            case 0xa6:
+            case 0xa7:
+            case 0xa8:
+            case 0xa9:
+            case 0xaa:
+            case 0xab:
+            case 0xac:
+            case 0xad:
+            case 0xae:
+            case 0xaf:
+            case 0xb0:
+            case 0xb1:
+            case 0xb2:
+            case 0xb3:
+            case 0xb4:
+            case 0xb5:
+            case 0xb6:
+            case 0xb7:
+            case 0xb8:
+            case 0xb9:
+            case 0xba:
+            case 0xbb:
+            case 0xbc:
+            case 0xbd:
+            case 0xbe:
+            case 0xbf:
+            {
+                return get_string(current & 0x1f);
+            }
+
+            case 0xd9: // str 8
+            {
+                return get_string(get_number<uint8_t>());
+            }
+
+            case 0xda: // str 16
+            {
+                return get_string(get_number<uint16_t>());
+            }
+
+            case 0xdb: // str 32
+            {
+                return get_string(get_number<uint32_t>());
+            }
+
+            default:
+            {
+                std::stringstream ss;
+                ss << std::setw(2) << std::setfill('0') << std::hex << current;
+                JSON_THROW(parse_error::create(113, chars_read,
+                                               "expected a MessagePack string; last byte: 0x" + ss.str()));
+            }
+        }
+    }
+
+    template<typename NumberType>
+    BasicJsonType get_msgpack_array(const NumberType len)
+    {
+        BasicJsonType result = value_t::array;
+        std::generate_n(std::back_inserter(*result.m_value.array), len, [this]()
+        {
+            return parse_msgpack_internal();
+        });
+        return result;
+    }
+
+    template<typename NumberType>
+    BasicJsonType get_msgpack_object(const NumberType len)
+    {
+        BasicJsonType result = value_t::object;
+        std::generate_n(std::inserter(*result.m_value.object,
+                                      result.m_value.object->end()),
+                        len, [this]()
+        {
+            get();
+            auto key = get_msgpack_string();
+            auto val = parse_msgpack_internal();
+            return std::make_pair(std::move(key), std::move(val));
+        });
+        return result;
+    }
+
+    /*!
+    @brief check if input ended
+    @throw parse_error.110 if input ended
+    */
+    void check_eof(const bool expect_eof = false) const
+    {
+        if (expect_eof)
+        {
+            if (JSON_UNLIKELY(current != std::char_traits<char>::eof()))
+            {
+                JSON_THROW(parse_error::create(110, chars_read, "expected end of input"));
+            }
+        }
+        else
+        {
+            if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
+            {
+                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
+            }
+        }
+    }
+
+  private:
+    /// input adapter
+    input_adapter_t ia = nullptr;
+
+    /// the current character
+    int current = std::char_traits<char>::eof();
+
+    /// the number of characters read
+    std::size_t chars_read = 0;
+
+    /// whether we can assume little endianess
+    const bool is_little_endian = little_endianess();
+};
+
+/*!
+@brief serialization to CBOR and MessagePack values
+*/
+template<typename BasicJsonType, typename CharType>
+class binary_writer
+{
+  public:
+    /*!
+    @brief create a binary writer
+
+    @param[in] adapter  output adapter to write to
+    */
+    explicit binary_writer(output_adapter_t<CharType> adapter) : oa(adapter)
+    {
+        assert(oa);
+    }
+
+    /*!
+    @brief[in] j  JSON value to serialize
+    */
+    void write_cbor(const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::null:
+            {
+                oa->write_character(static_cast<CharType>(0xf6));
+                break;
+            }
+
+            case value_t::boolean:
+            {
+                oa->write_character(j.m_value.boolean
+                                    ? static_cast<CharType>(0xf5)
+                                    : static_cast<CharType>(0xf4));
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                if (j.m_value.number_integer >= 0)
+                {
+                    // CBOR does not differentiate between positive signed
+                    // integers and unsigned integers. Therefore, we used the
+                    // code from the value_t::number_unsigned case here.
+                    if (j.m_value.number_integer <= 0x17)
+                    {
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x18));
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x19));
+                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x1a));
+                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    }
+                    else
+                    {
+                        oa->write_character(static_cast<CharType>(0x1b));
+                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    }
+                }
+                else
+                {
+                    // The conversions below encode the sign in the first
+                    // byte, and the value is converted to a positive number.
+                    const auto positive_number = -1 - j.m_value.number_integer;
+                    if (j.m_value.number_integer >= -24)
+                    {
+                        write_number(static_cast<uint8_t>(0x20 + positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x38));
+                        write_number(static_cast<uint8_t>(positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x39));
+                        write_number(static_cast<uint16_t>(positive_number));
+                    }
+                    else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        oa->write_character(static_cast<CharType>(0x3a));
+                        write_number(static_cast<uint32_t>(positive_number));
+                    }
+                    else
+                    {
+                        oa->write_character(static_cast<CharType>(0x3b));
+                        write_number(static_cast<uint64_t>(positive_number));
+                    }
+                }
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                if (j.m_value.number_unsigned <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    oa->write_character(static_cast<CharType>(0x18));
+                    write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    oa->write_character(static_cast<CharType>(0x19));
+                    write_number(static_cast<uint16_t>(j.m_value.number_unsigned));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    oa->write_character(static_cast<CharType>(0x1a));
+                    write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
+                }
+                else
+                {
+                    oa->write_character(static_cast<CharType>(0x1b));
+                    write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
+                }
+                break;
+            }
+
+            case value_t::number_float: // Double-Precision Float
+            {
+                oa->write_character(static_cast<CharType>(0xfb));
+                write_number(j.m_value.number_float);
+                break;
+            }
+
+            case value_t::string:
+            {
+                // step 1: write control byte and the string length
+                const auto N = j.m_value.string->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0x60 + N));
+                }
+                else if (N <= 0xff)
+                {
+                    oa->write_character(static_cast<CharType>(0x78));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= 0xffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x79));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 0xffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x7a));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= 0xffffffffffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x7b));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write the string
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
+            }
+
+            case value_t::array:
+            {
+                // step 1: write control byte and the array size
+                const auto N = j.m_value.array->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0x80 + N));
+                }
+                else if (N <= 0xff)
+                {
+                    oa->write_character(static_cast<CharType>(0x98));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= 0xffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x99));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 0xffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x9a));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= 0xffffffffffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0x9b));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_cbor(el);
+                }
+                break;
+            }
+
+            case value_t::object:
+            {
+                // step 1: write control byte and the object size
+                const auto N = j.m_value.object->size();
+                if (N <= 0x17)
+                {
+                    write_number(static_cast<uint8_t>(0xa0 + N));
+                }
+                else if (N <= 0xff)
+                {
+                    oa->write_character(static_cast<CharType>(0xb8));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= 0xffff)
+                {
+                    oa->write_character(static_cast<CharType>(0xb9));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 0xffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0xba));
+                    write_number(static_cast<uint32_t>(N));
+                }
+                // LCOV_EXCL_START
+                else if (N <= 0xffffffffffffffff)
+                {
+                    oa->write_character(static_cast<CharType>(0xbb));
+                    write_number(static_cast<uint64_t>(N));
+                }
+                // LCOV_EXCL_STOP
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_cbor(el.first);
+                    write_cbor(el.second);
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+    /*!
+    @brief[in] j  JSON value to serialize
+    */
+    void write_msgpack(const BasicJsonType& j)
+    {
+        switch (j.type())
+        {
+            case value_t::null: // nil
+            {
+                oa->write_character(static_cast<CharType>(0xc0));
+                break;
+            }
+
+            case value_t::boolean: // true and false
+            {
+                oa->write_character(j.m_value.boolean
+                                    ? static_cast<CharType>(0xc3)
+                                    : static_cast<CharType>(0xc2));
+                break;
+            }
+
+            case value_t::number_integer:
+            {
+                if (j.m_value.number_integer >= 0)
+                {
+                    // MessagePack does not differentiate between positive
+                    // signed integers and unsigned integers. Therefore, we used
+                    // the code from the value_t::number_unsigned case here.
+                    if (j.m_value.number_unsigned < 128)
+                    {
+                        // positive fixnum
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                    {
+                        // uint 8
+                        oa->write_character(static_cast<CharType>(0xcc));
+                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                    {
+                        // uint 16
+                        oa->write_character(static_cast<CharType>(0xcd));
+                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                    {
+                        // uint 32
+                        oa->write_character(static_cast<CharType>(0xce));
+                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                    {
+                        // uint 64
+                        oa->write_character(static_cast<CharType>(0xcf));
+                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                    }
+                }
+                else
+                {
+                    if (j.m_value.number_integer >= -32)
+                    {
+                        // negative fixnum
+                        write_number(static_cast<int8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
+                    {
+                        // int 8
+                        oa->write_character(static_cast<CharType>(0xd0));
+                        write_number(static_cast<int8_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
+                    {
+                        // int 16
+                        oa->write_character(static_cast<CharType>(0xd1));
+                        write_number(static_cast<int16_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
+                    {
+                        // int 32
+                        oa->write_character(static_cast<CharType>(0xd2));
+                        write_number(static_cast<int32_t>(j.m_value.number_integer));
+                    }
+                    else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and
+                             j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
+                    {
+                        // int 64
+                        oa->write_character(static_cast<CharType>(0xd3));
+                        write_number(static_cast<int64_t>(j.m_value.number_integer));
+                    }
+                }
+                break;
+            }
+
+            case value_t::number_unsigned:
+            {
+                if (j.m_value.number_unsigned < 128)
+                {
+                    // positive fixnum
+                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
+                {
+                    // uint 8
+                    oa->write_character(static_cast<CharType>(0xcc));
+                    write_number(static_cast<uint8_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
+                {
+                    // uint 16
+                    oa->write_character(static_cast<CharType>(0xcd));
+                    write_number(static_cast<uint16_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
+                {
+                    // uint 32
+                    oa->write_character(static_cast<CharType>(0xce));
+                    write_number(static_cast<uint32_t>(j.m_value.number_integer));
+                }
+                else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
+                {
+                    // uint 64
+                    oa->write_character(static_cast<CharType>(0xcf));
+                    write_number(static_cast<uint64_t>(j.m_value.number_integer));
+                }
+                break;
+            }
+
+            case value_t::number_float: // float 64
+            {
+                oa->write_character(static_cast<CharType>(0xcb));
+                write_number(j.m_value.number_float);
+                break;
+            }
+
+            case value_t::string:
+            {
+                // step 1: write control byte and the string length
+                const auto N = j.m_value.string->size();
+                if (N <= 31)
+                {
+                    // fixstr
+                    write_number(static_cast<uint8_t>(0xa0 | N));
+                }
+                else if (N <= 255)
+                {
+                    // str 8
+                    oa->write_character(static_cast<CharType>(0xd9));
+                    write_number(static_cast<uint8_t>(N));
+                }
+                else if (N <= 65535)
+                {
+                    // str 16
+                    oa->write_character(static_cast<CharType>(0xda));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 4294967295)
+                {
+                    // str 32
+                    oa->write_character(static_cast<CharType>(0xdb));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write the string
+                oa->write_characters(
+                    reinterpret_cast<const CharType*>(j.m_value.string->c_str()),
+                    j.m_value.string->size());
+                break;
+            }
+
+            case value_t::array:
+            {
+                // step 1: write control byte and the array size
+                const auto N = j.m_value.array->size();
+                if (N <= 15)
+                {
+                    // fixarray
+                    write_number(static_cast<uint8_t>(0x90 | N));
+                }
+                else if (N <= 0xffff)
+                {
+                    // array 16
+                    oa->write_character(static_cast<CharType>(0xdc));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 0xffffffff)
+                {
+                    // array 32
+                    oa->write_character(static_cast<CharType>(0xdd));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.array)
+                {
+                    write_msgpack(el);
+                }
+                break;
+            }
+
+            case value_t::object:
+            {
+                // step 1: write control byte and the object size
+                const auto N = j.m_value.object->size();
+                if (N <= 15)
+                {
+                    // fixmap
+                    write_number(static_cast<uint8_t>(0x80 | (N & 0xf)));
+                }
+                else if (N <= 65535)
+                {
+                    // map 16
+                    oa->write_character(static_cast<CharType>(0xde));
+                    write_number(static_cast<uint16_t>(N));
+                }
+                else if (N <= 4294967295)
+                {
+                    // map 32
+                    oa->write_character(static_cast<CharType>(0xdf));
+                    write_number(static_cast<uint32_t>(N));
+                }
+
+                // step 2: write each element
+                for (const auto& el : *j.m_value.object)
+                {
+                    write_msgpack(el.first);
+                    write_msgpack(el.second);
+                }
+                break;
+            }
+
+            default:
+                break;
+        }
+    }
+
+  private:
+    /*
+    @brief write a number to output input
+
+    @param[in] n number of type @a NumberType
+    @tparam NumberType the type of the number
+
+    @note This function needs to respect the system's endianess, because bytes
+          in CBOR and MessagePack are stored in network order (big endian) and
+          therefore need reordering on little endian systems.
+    */
+    template<typename NumberType> void write_number(NumberType n)
+    {
+        // step 1: write number to array of length NumberType
+        std::array<CharType, sizeof(NumberType)> vec;
+        std::memcpy(vec.data(), &n, sizeof(NumberType));
+
+        // step 2: write array to output (with possible reordering)
+        if (is_little_endian)
+        {
+            // reverse byte order prior to conversion if necessary
+            std::reverse(vec.begin(), vec.end());
+        }
+
+        oa->write_characters(vec.data(), sizeof(NumberType));
+    }
+
+  private:
+    /// whether we can assume little endianess
+    const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess();
+
+    /// the output
+    output_adapter_t<CharType> oa = nullptr;
+};
+
+///////////////////
+// serialization //
+///////////////////
+
+template<typename BasicJsonType>
+class serializer
+{
+    using string_t = typename BasicJsonType::string_t;
+    using number_float_t = typename BasicJsonType::number_float_t;
+    using number_integer_t = typename BasicJsonType::number_integer_t;
+    using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
+  public:
+    /*!
+    @param[in] s  output stream to serialize to
+    @param[in] ichar  indentation character to use
+    */
+    serializer(output_adapter_t<char> s, const char ichar)
+        : o(std::move(s)), loc(std::localeconv()),
+          thousands_sep(loc->thousands_sep == nullptr ? '\0' : loc->thousands_sep[0]),
+          decimal_point(loc->decimal_point == nullptr ? '\0' : loc->decimal_point[0]),
+          indent_char(ichar), indent_string(512, indent_char) {}
+
+    // delete because of pointer members
+    serializer(const serializer&) = delete;
+    serializer& operator=(const serializer&) = delete;
+
+    /*!
+    @brief internal implementation of the serialization function
+
+    This function is called by the public member function dump and organizes
+    the serialization internally. The indentation level is propagated as
+    additional parameter. In case of arrays and objects, the function is
+    called recursively.
+
+    - strings and object keys are escaped using `escape_string()`
+    - integer numbers are converted implicitly via `operator<<`
+    - floating-point numbers are converted to a string using `"%g"` format
+
+    @param[in] val             value to serialize
+    @param[in] pretty_print    whether the output shall be pretty-printed
+    @param[in] indent_step     the indent level
+    @param[in] current_indent  the current indent level (only used internally)
+    */
+    void dump(const BasicJsonType& val, const bool pretty_print,
+              const bool ensure_ascii,
+              const unsigned int indent_step,
+              const unsigned int current_indent = 0)
+    {
+        switch (val.m_type)
+        {
+            case value_t::object:
+            {
+                if (val.m_value.object->empty())
+                {
+                    o->write_characters("{}", 2);
+                    return;
+                }
+
+                if (pretty_print)
+                {
+                    o->write_characters("{\n", 2);
+
+                    // variable to hold indentation for recursive calls
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
+
+                    // first n-1 elements
+                    auto i = val.m_value.object->cbegin();
+                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
+                    {
+                        o->write_characters(indent_string.c_str(), new_indent);
+                        o->write_character('\"');
+                        dump_escaped(i->first, ensure_ascii);
+                        o->write_characters("\": ", 3);
+                        dump(i->second, true, ensure_ascii, indent_step, new_indent);
+                        o->write_characters(",\n", 2);
+                    }
+
+                    // last element
+                    assert(i != val.m_value.object->cend());
+                    assert(std::next(i) == val.m_value.object->cend());
+                    o->write_characters(indent_string.c_str(), new_indent);
+                    o->write_character('\"');
+                    dump_escaped(i->first, ensure_ascii);
+                    o->write_characters("\": ", 3);
+                    dump(i->second, true, ensure_ascii, indent_step, new_indent);
+
+                    o->write_character('\n');
+                    o->write_characters(indent_string.c_str(), current_indent);
+                    o->write_character('}');
+                }
+                else
+                {
+                    o->write_character('{');
+
+                    // first n-1 elements
+                    auto i = val.m_value.object->cbegin();
+                    for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
+                    {
+                        o->write_character('\"');
+                        dump_escaped(i->first, ensure_ascii);
+                        o->write_characters("\":", 2);
+                        dump(i->second, false, ensure_ascii, indent_step, current_indent);
+                        o->write_character(',');
+                    }
+
+                    // last element
+                    assert(i != val.m_value.object->cend());
+                    assert(std::next(i) == val.m_value.object->cend());
+                    o->write_character('\"');
+                    dump_escaped(i->first, ensure_ascii);
+                    o->write_characters("\":", 2);
+                    dump(i->second, false, ensure_ascii, indent_step, current_indent);
+
+                    o->write_character('}');
+                }
+
+                return;
+            }
+
+            case value_t::array:
+            {
+                if (val.m_value.array->empty())
+                {
+                    o->write_characters("[]", 2);
+                    return;
+                }
+
+                if (pretty_print)
+                {
+                    o->write_characters("[\n", 2);
+
+                    // variable to hold indentation for recursive calls
+                    const auto new_indent = current_indent + indent_step;
+                    if (JSON_UNLIKELY(indent_string.size() < new_indent))
+                    {
+                        indent_string.resize(indent_string.size() * 2, ' ');
+                    }
+
+                    // first n-1 elements
+                    for (auto i = val.m_value.array->cbegin();
+                            i != val.m_value.array->cend() - 1; ++i)
+                    {
+                        o->write_characters(indent_string.c_str(), new_indent);
+                        dump(*i, true, ensure_ascii, indent_step, new_indent);
+                        o->write_characters(",\n", 2);
+                    }
+
+                    // last element
+                    assert(not val.m_value.array->empty());
+                    o->write_characters(indent_string.c_str(), new_indent);
+                    dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent);
+
+                    o->write_character('\n');
+                    o->write_characters(indent_string.c_str(), current_indent);
+                    o->write_character(']');
+                }
+                else
+                {
+                    o->write_character('[');
+
+                    // first n-1 elements
+                    for (auto i = val.m_value.array->cbegin();
+                            i != val.m_value.array->cend() - 1; ++i)
+                    {
+                        dump(*i, false, ensure_ascii, indent_step, current_indent);
+                        o->write_character(',');
+                    }
+
+                    // last element
+                    assert(not val.m_value.array->empty());
+                    dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent);
+
+                    o->write_character(']');
+                }
+
+                return;
+            }
+
+            case value_t::string:
+            {
+                o->write_character('\"');
+                dump_escaped(*val.m_value.string, ensure_ascii);
+                o->write_character('\"');
+                return;
+            }
+
+            case value_t::boolean:
+            {
+                if (val.m_value.boolean)
+                {
+                    o->write_characters("true", 4);
+                }
+                else
+                {
+                    o->write_characters("false", 5);
+                }
+                return;
+            }
+
+            case value_t::number_integer:
+            {
+                dump_integer(val.m_value.number_integer);
+                return;
+            }
+
+            case value_t::number_unsigned:
+            {
+                dump_integer(val.m_value.number_unsigned);
+                return;
+            }
+
+            case value_t::number_float:
+            {
+                dump_float(val.m_value.number_float);
+                return;
+            }
+
+            case value_t::discarded:
+            {
+                o->write_characters("<discarded>", 11);
+                return;
+            }
+
+            case value_t::null:
+            {
+                o->write_characters("null", 4);
+                return;
+            }
+        }
+    }
+
+  private:
+    /*!
+    @brief returns the number of expected bytes following in UTF-8 string
+
+    @param[in]  u  the first byte of a UTF-8 string
+    @return  the number of expected bytes following
+    */
+    static constexpr std::size_t bytes_following(const uint8_t u)
+    {
+        return ((0 <= u and u <= 127) ? 0
+                : ((192 <= u and u <= 223) ? 1
+                   : ((224 <= u and u <= 239) ? 2
+                      : ((240 <= u and u <= 247) ? 3 : std::string::npos))));
+    }
+
+    /*!
+    @brief calculates the extra space to escape a JSON string
+
+    @param[in] s  the string to escape
+    @param[in] ensure_ascii  whether to escape non-ASCII characters with
+                             \uXXXX sequences
+    @return the number of characters required to escape string @a s
+
+    @complexity Linear in the length of string @a s.
+    */
+    static std::size_t extra_space(const string_t& s,
+                                   const bool ensure_ascii) noexcept
+    {
+        std::size_t res = 0;
+
+        for (std::size_t i = 0; i < s.size(); ++i)
+        {
+            switch (s[i])
+            {
+                // control characters that can be escaped with a backslash
+                case '"':
+                case '\\':
+                case '\b':
+                case '\f':
+                case '\n':
+                case '\r':
+                case '\t':
+                {
+                    // from c (1 byte) to \x (2 bytes)
+                    res += 1;
+                    break;
+                }
+
+                // control characters that need \uxxxx escaping
+                case 0x00:
+                case 0x01:
+                case 0x02:
+                case 0x03:
+                case 0x04:
+                case 0x05:
+                case 0x06:
+                case 0x07:
+                case 0x0b:
+                case 0x0e:
+                case 0x0f:
+                case 0x10:
+                case 0x11:
+                case 0x12:
+                case 0x13:
+                case 0x14:
+                case 0x15:
+                case 0x16:
+                case 0x17:
+                case 0x18:
+                case 0x19:
+                case 0x1a:
+                case 0x1b:
+                case 0x1c:
+                case 0x1d:
+                case 0x1e:
+                case 0x1f:
+                {
+                    // from c (1 byte) to \uxxxx (6 bytes)
+                    res += 5;
+                    break;
+                }
+
+                default:
+                {
+                    if (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F))
+                    {
+                        const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
+                        if (bytes == std::string::npos)
+                        {
+                            // invalid characters are treated as is, so no
+                            // additional space will be used
+                            break;
+                        }
+
+                        if (bytes == 3)
+                        {
+                            // codepoints that need 4 bytes (i.e., 3 additional
+                            // bytes) in UTF-8 need a surrogate pair when \u
+                            // escaping is used: from 4 bytes to \uxxxx\uxxxx
+                            // (12 bytes)
+                            res += (12 - bytes - 1);
+                        }
+                        else
+                        {
+                            // from x bytes to \uxxxx (6 bytes)
+                            res += (6 - bytes - 1);
+                        }
+
+                        // skip the additional bytes
+                        i += bytes;
+                    }
+                    break;
+                }
+            }
+        }
+
+        return res;
+    }
+
+    static void escape_codepoint(int codepoint, string_t& result, std::size_t& pos)
+    {
+        // expecting a proper codepoint
+        assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
+
+        // the last written character was the backslash before the 'u'
+        assert(result[pos] == '\\');
+
+        // write the 'u'
+        result[++pos] = 'u';
+
+        // convert a number 0..15 to its hex representation (0..f)
+        static const std::array<char, 16> hexify =
+        {
+            {
+                '0', '1', '2', '3', '4', '5', '6', '7',
+                '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
+            }
+        };
+
+        if (codepoint < 0x10000)
+        {
+            // codepoints U+0000..U+FFFF can be represented as \uxxxx.
+            result[++pos] = hexify[(codepoint >> 12) & 0x0F];
+            result[++pos] = hexify[(codepoint >> 8) & 0x0F];
+            result[++pos] = hexify[(codepoint >> 4) & 0x0F];
+            result[++pos] = hexify[codepoint & 0x0F];
+        }
+        else
+        {
+            // codepoints U+10000..U+10FFFF need a surrogate pair to be
+            // represented as \uxxxx\uxxxx.
+            // http://www.unicode.org/faq/utf_bom.html#utf16-4
+            codepoint -= 0x10000;
+            const int high_surrogate = 0xD800 | ((codepoint >> 10) & 0x3FF);
+            const int low_surrogate = 0xDC00 | (codepoint & 0x3FF);
+            result[++pos] = hexify[(high_surrogate >> 12) & 0x0F];
+            result[++pos] = hexify[(high_surrogate >> 8) & 0x0F];
+            result[++pos] = hexify[(high_surrogate >> 4) & 0x0F];
+            result[++pos] = hexify[high_surrogate & 0x0F];
+            ++pos;  // backslash is already in output
+            result[++pos] = 'u';
+            result[++pos] = hexify[(low_surrogate >> 12) & 0x0F];
+            result[++pos] = hexify[(low_surrogate >> 8) & 0x0F];
+            result[++pos] = hexify[(low_surrogate >> 4) & 0x0F];
+            result[++pos] = hexify[low_surrogate & 0x0F];
+        }
+
+        ++pos;
+    }
+
+    /*!
+    @brief dump escaped string
+
+    Escape a string by replacing certain special characters by a sequence of an
+    escape character (backslash) and another character and other control
+    characters by a sequence of "\u" followed by a four-digit hex
+    representation. The escaped string is written to output stream @a o.
+
+    @param[in] s  the string to escape
+    @param[in] ensure_ascii  whether to escape non-ASCII characters with
+                             \uXXXX sequences
+
+    @complexity Linear in the length of string @a s.
+    */
+    void dump_escaped(const string_t& s, const bool ensure_ascii) const
+    {
+        const auto space = extra_space(s, ensure_ascii);
+        if (space == 0)
+        {
+            o->write_characters(s.c_str(), s.size());
+            return;
+        }
+
+        // create a result string of necessary size
+        string_t result(s.size() + space, '\\');
+        std::size_t pos = 0;
+
+        for (std::size_t i = 0; i < s.size(); ++i)
+        {
+            switch (s[i])
+            {
+                case '"': // quotation mark (0x22)
+                {
+                    result[pos + 1] = '"';
+                    pos += 2;
+                    break;
+                }
+
+                case '\\': // reverse solidus (0x5c)
+                {
+                    // nothing to change
+                    pos += 2;
+                    break;
+                }
+
+                case '\b': // backspace (0x08)
+                {
+                    result[pos + 1] = 'b';
+                    pos += 2;
+                    break;
+                }
+
+                case '\f': // formfeed (0x0c)
+                {
+                    result[pos + 1] = 'f';
+                    pos += 2;
+                    break;
+                }
+
+                case '\n': // newline (0x0a)
+                {
+                    result[pos + 1] = 'n';
+                    pos += 2;
+                    break;
+                }
+
+                case '\r': // carriage return (0x0d)
+                {
+                    result[pos + 1] = 'r';
+                    pos += 2;
+                    break;
+                }
+
+                case '\t': // horizontal tab (0x09)
+                {
+                    result[pos + 1] = 't';
+                    pos += 2;
+                    break;
+                }
+
+                default:
+                {
+                    // escape control characters (0x00..0x1F) or, if
+                    // ensure_ascii parameter is used, non-ASCII characters
+                    if ((0x00 <= s[i] and s[i] <= 0x1F) or
+                            (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F)))
+                    {
+                        const auto bytes = bytes_following(static_cast<uint8_t>(s[i]));
+                        if (bytes == std::string::npos)
+                        {
+                            // copy invalid character as is
+                            result[pos++] = s[i];
+                            break;
+                        }
+
+                        // check that the additional bytes are present
+                        assert(i + bytes < s.size());
+
+                        // to use \uxxxx escaping, we first need to caluclate
+                        // the codepoint from the UTF-8 bytes
+                        int codepoint = 0;
+
+                        assert(0 <= bytes and bytes <= 3);
+                        switch (bytes)
+                        {
+                            case 0:
+                            {
+                                codepoint = s[i] & 0xFF;
+                                break;
+                            }
+
+                            case 1:
+                            {
+                                codepoint = ((s[i] & 0x3F) << 6)
+                                            + (s[i + 1] & 0x7F);
+                                break;
+                            }
+
+                            case 2:
+                            {
+                                codepoint = ((s[i] & 0x1F) << 12)
+                                            + ((s[i + 1] & 0x7F) << 6)
+                                            + (s[i + 2] & 0x7F);
+                                break;
+                            }
+
+                            case 3:
+                            {
+                                codepoint = ((s[i] & 0xF) << 18)
+                                            + ((s[i + 1] & 0x7F) << 12)
+                                            + ((s[i + 2] & 0x7F) << 6)
+                                            + (s[i + 3] & 0x7F);
+                                break;
+                            }
+
+                            default:
+                                break;  // LCOV_EXCL_LINE
+                        }
+
+                        escape_codepoint(codepoint, result, pos);
+                        i += bytes;
+                    }
+                    else
+                    {
+                        // all other characters are added as-is
+                        result[pos++] = s[i];
+                    }
+                    break;
+                }
+            }
+        }
+
+        assert(pos == result.size());
+        o->write_characters(result.c_str(), result.size());
+    }
+
+    /*!
+    @brief dump an integer
+
+    Dump a given integer to output stream @a o. Works internally with
+    @a number_buffer.
+
+    @param[in] x  integer number (signed or unsigned) to dump
+    @tparam NumberType either @a number_integer_t or @a number_unsigned_t
+    */
+    template <
+        typename NumberType,
+        detail::enable_if_t<std::is_same<NumberType, number_unsigned_t>::value or
+                            std::is_same<NumberType, number_integer_t>::value,
+                            int> = 0 >
+    void dump_integer(NumberType x)
+    {
+        // special case for "0"
+        if (x == 0)
+        {
+            o->write_character('0');
+            return;
+        }
+
+        const bool is_negative = x < 0;
+        std::size_t i = 0;
+
+        // spare 1 byte for '\0'
+        while (x != 0 and i < number_buffer.size() - 1)
+        {
+            const auto digit = std::labs(static_cast<long>(x % 10));
+            number_buffer[i++] = static_cast<char>('0' + digit);
+            x /= 10;
+        }
+
+        // make sure the number has been processed completely
+        assert(x == 0);
+
+        if (is_negative)
+        {
+            // make sure there is capacity for the '-'
+            assert(i < number_buffer.size() - 2);
+            number_buffer[i++] = '-';
+        }
+
+        std::reverse(number_buffer.begin(), number_buffer.begin() + i);
+        o->write_characters(number_buffer.data(), i);
+    }
+
+    /*!
+    @brief dump a floating-point number
+
+    Dump a given floating-point number to output stream @a o. Works internally
+    with @a number_buffer.
+
+    @param[in] x  floating-point number to dump
+    */
+    void dump_float(number_float_t x)
+    {
+        // NaN / inf
+        if (not std::isfinite(x) or std::isnan(x))
+        {
+            o->write_characters("null", 4);
+            return;
+        }
+
+        // special case for 0.0 and -0.0
+        if (x == 0)
+        {
+            if (std::signbit(x))
+            {
+                o->write_characters("-0.0", 4);
+            }
+            else
+            {
+                o->write_characters("0.0", 3);
+            }
+            return;
+        }
+
+        // get number of digits for a text -> float -> text round-trip
+        static constexpr auto d = std::numeric_limits<number_float_t>::digits10;
+
+        // the actual conversion
+        std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), "%.*g", d, x);
+
+        // negative value indicates an error
+        assert(len > 0);
+        // check if buffer was large enough
+        assert(static_cast<std::size_t>(len) < number_buffer.size());
+
+        // erase thousands separator
+        if (thousands_sep != '\0')
+        {
+            const auto end = std::remove(number_buffer.begin(),
+                                         number_buffer.begin() + len, thousands_sep);
+            std::fill(end, number_buffer.end(), '\0');
+            assert((end - number_buffer.begin()) <= len);
+            len = (end - number_buffer.begin());
+        }
+
+        // convert decimal point to '.'
+        if (decimal_point != '\0' and decimal_point != '.')
+        {
+            const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point);
+            if (dec_pos != number_buffer.end())
+            {
+                *dec_pos = '.';
+            }
+        }
+
+        o->write_characters(number_buffer.data(), static_cast<std::size_t>(len));
+
+        // determine if need to append ".0"
+        const bool value_is_int_like =
+            std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1,
+                         [](char c)
+        {
+            return (c == '.' or c == 'e');
+        });
+
+        if (value_is_int_like)
+        {
+            o->write_characters(".0", 2);
+        }
+    }
+
+  private:
+    /// the output of the serializer
+    output_adapter_t<char> o = nullptr;
+
+    /// a (hopefully) large enough character buffer
+    std::array<char, 64> number_buffer{{}};
+
+    /// the locale
+    const std::lconv* loc = nullptr;
+    /// the locale's thousand separator character
+    const char thousands_sep = '\0';
+    /// the locale's decimal point character
+    const char decimal_point = '\0';
+
+    /// the indentation character
+    const char indent_char;
+
+    /// the indentation string
+    string_t indent_string;
+};
+
+template<typename BasicJsonType>
+class json_ref
+{
+  public:
+    using value_type = BasicJsonType;
+
+    json_ref(value_type&& value)
+        : owned_value(std::move(value)),
+          value_ref(&owned_value),
+          is_rvalue(true)
+    {}
+
+    json_ref(const value_type& value)
+        : value_ref(const_cast<value_type*>(&value)),
+          is_rvalue(false)
+    {}
+
+    json_ref(std::initializer_list<json_ref> init)
+        : owned_value(init),
+          value_ref(&owned_value),
+          is_rvalue(true)
+    {}
+
+    template <class... Args>
+    json_ref(Args... args)
+        : owned_value(std::forward<Args>(args)...),
+          value_ref(&owned_value),
+          is_rvalue(true)
+    {}
+
+    // class should be movable only
+    json_ref(json_ref&&) = default;
+    json_ref(const json_ref&) = delete;
+    json_ref& operator=(const json_ref&) = delete;
+
+    value_type moved_or_copied() const
+    {
+        if (is_rvalue)
+        {
+            return std::move(*value_ref);
+        }
+        return *value_ref;
+    }
+
+    value_type const& operator*() const
+    {
+        return *static_cast<value_type const*>(value_ref);
+    }
+
+    value_type const* operator->() const
+    {
+        return static_cast<value_type const*>(value_ref);
+    }
+
+  private:
+    mutable value_type owned_value = nullptr;
+    value_type* value_ref = nullptr;
+    const bool is_rvalue;
+};
+
+} // namespace detail
 
 /// namespace to hold default `to_json` / `from_json` functions
 namespace
@@ -1252,7 +6881,7 @@ This serializer ignores the template arguments and uses ADL
 ([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl))
 for serialization.
 */
-template<typename = void, typename = void>
+template<typename, typename>
 struct adl_serializer
 {
     /*!
@@ -1288,6 +6917,326 @@ struct adl_serializer
     }
 };
 
+/*!
+@brief JSON Pointer
+
+A JSON pointer defines a string syntax for identifying a specific value
+within a JSON document. It can be used with functions `at` and
+`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
+
+@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
+
+@since version 2.0.0
+*/
+class json_pointer
+{
+    /// allow basic_json to access private members
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    friend class basic_json;
+
+  public:
+    /*!
+    @brief create JSON pointer
+
+    Create a JSON pointer according to the syntax described in
+    [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3).
+
+    @param[in] s  string representing the JSON pointer; if omitted, the empty
+                  string is assumed which references the whole JSON value
+
+    @throw parse_error.107 if the given JSON pointer @a s is nonempty and
+    does not begin with a slash (`/`); see example below
+
+    @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s
+    is not followed by `0` (representing `~`) or `1` (representing `/`);
+    see example below
+
+    @liveexample{The example shows the construction several valid JSON
+    pointers as well as the exceptional behavior.,json_pointer}
+
+    @since version 2.0.0
+    */
+    explicit json_pointer(const std::string& s = "") : reference_tokens(split(s)) {}
+
+    /*!
+    @brief return a string representation of the JSON pointer
+
+    @invariant For each JSON pointer `ptr`, it holds:
+    @code {.cpp}
+    ptr == json_pointer(ptr.to_string());
+    @endcode
+
+    @return a string representation of the JSON pointer
+
+    @liveexample{The example shows the result of `to_string`.,
+    json_pointer__to_string}
+
+    @since version 2.0.0
+    */
+    std::string to_string() const noexcept
+    {
+        return std::accumulate(reference_tokens.begin(), reference_tokens.end(),
+                               std::string{},
+                               [](const std::string & a, const std::string & b)
+        {
+            return a + "/" + escape(b);
+        });
+    }
+
+    /// @copydoc to_string()
+    operator std::string() const
+    {
+        return to_string();
+    }
+
+  private:
+    /*!
+    @brief remove and return last reference pointer
+    @throw out_of_range.405 if JSON pointer has no parent
+    */
+    std::string pop_back()
+    {
+        if (JSON_UNLIKELY(is_root()))
+        {
+            JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent"));
+        }
+
+        auto last = reference_tokens.back();
+        reference_tokens.pop_back();
+        return last;
+    }
+
+    /// return whether pointer points to the root document
+    bool is_root() const
+    {
+        return reference_tokens.empty();
+    }
+
+    json_pointer top() const
+    {
+        if (JSON_UNLIKELY(is_root()))
+        {
+            JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent"));
+        }
+
+        json_pointer result = *this;
+        result.reference_tokens = {reference_tokens[0]};
+        return result;
+    }
+
+
+    /*!
+    @brief create and return a reference to the pointed to value
+
+    @complexity Linear in the number of reference tokens.
+
+    @throw parse_error.109 if array index is not a number
+    @throw type_error.313 if value cannot be unflattened
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    NLOHMANN_BASIC_JSON_TPL& get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const;
+
+    /*!
+    @brief return a reference to the pointed to value
+
+    @note This version does not throw if a value is not present, but tries to
+          create nested values instead. For instance, calling this function
+          with pointer `"/this/that"` on a null value is equivalent to calling
+          `operator[]("this").operator[]("that")` on that value, effectively
+          changing the null value to an object.
+
+    @param[in] ptr  a JSON value
+
+    @return reference to the JSON value pointed to by the JSON pointer
+
+    @complexity Linear in the length of the JSON pointer.
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    NLOHMANN_BASIC_JSON_TPL& get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const;
+
+    /*!
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    NLOHMANN_BASIC_JSON_TPL& get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const;
+
+    /*!
+    @brief return a const reference to the pointed to value
+
+    @param[in] ptr  a JSON value
+
+    @return const reference to the JSON value pointed to by the JSON
+    pointer
+
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    const NLOHMANN_BASIC_JSON_TPL& get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const;
+
+    /*!
+    @throw parse_error.106   if an array index begins with '0'
+    @throw parse_error.109   if an array index was not a number
+    @throw out_of_range.402  if the array index '-' is used
+    @throw out_of_range.404  if the JSON pointer can not be resolved
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    const NLOHMANN_BASIC_JSON_TPL& get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const;
+
+    /*!
+    @brief split the string input to reference tokens
+
+    @note This function is only called by the json_pointer constructor.
+          All exceptions below are documented there.
+
+    @throw parse_error.107  if the pointer is not empty or begins with '/'
+    @throw parse_error.108  if character '~' is not followed by '0' or '1'
+    */
+    static std::vector<std::string> split(const std::string& reference_string)
+    {
+        std::vector<std::string> result;
+
+        // special case: empty reference string -> no reference tokens
+        if (reference_string.empty())
+        {
+            return result;
+        }
+
+        // check if nonempty reference string begins with slash
+        if (JSON_UNLIKELY(reference_string[0] != '/'))
+        {
+            JSON_THROW(detail::parse_error::create(107, 1,
+                                                   "JSON pointer must be empty or begin with '/' - was: '" +
+                                                   reference_string + "'"));
+        }
+
+        // extract the reference tokens:
+        // - slash: position of the last read slash (or end of string)
+        // - start: position after the previous slash
+        for (
+            // search for the first slash after the first character
+            std::size_t slash = reference_string.find_first_of('/', 1),
+            // set the beginning of the first reference token
+            start = 1;
+            // we can stop if start == string::npos+1 = 0
+            start != 0;
+            // set the beginning of the next reference token
+            // (will eventually be 0 if slash == std::string::npos)
+            start = slash + 1,
+            // find next slash
+            slash = reference_string.find_first_of('/', start))
+        {
+            // use the text between the beginning of the reference token
+            // (start) and the last slash (slash).
+            auto reference_token = reference_string.substr(start, slash - start);
+
+            // check reference tokens are properly escaped
+            for (std::size_t pos = reference_token.find_first_of('~');
+                    pos != std::string::npos;
+                    pos = reference_token.find_first_of('~', pos + 1))
+            {
+                assert(reference_token[pos] == '~');
+
+                // ~ must be followed by 0 or 1
+                if (JSON_UNLIKELY(pos == reference_token.size() - 1 or
+                                  (reference_token[pos + 1] != '0' and
+                                   reference_token[pos + 1] != '1')))
+                {
+                    JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'"));
+                }
+            }
+
+            // finally, store the reference token
+            unescape(reference_token);
+            result.push_back(reference_token);
+        }
+
+        return result;
+    }
+
+    /*!
+    @brief replace all occurrences of a substring by another string
+
+    @param[in,out] s  the string to manipulate; changed so that all
+                   occurrences of @a f are replaced with @a t
+    @param[in]     f  the substring to replace with @a t
+    @param[in]     t  the string to replace @a f
+
+    @pre The search string @a f must not be empty. **This precondition is
+    enforced with an assertion.**
+
+    @since version 2.0.0
+    */
+    static void replace_substring(std::string& s, const std::string& f,
+                                  const std::string& t)
+    {
+        assert(not f.empty());
+        for (auto pos = s.find(f);                // find first occurrence of f
+                pos != std::string::npos;         // make sure f was found
+                s.replace(pos, f.size(), t),      // replace with t, and
+                pos = s.find(f, pos + t.size()))  // find next occurrence of f
+        {}
+    }
+
+    /// escape "~"" to "~0" and "/" to "~1"
+    static std::string escape(std::string s)
+    {
+        replace_substring(s, "~", "~0");
+        replace_substring(s, "/", "~1");
+        return s;
+    }
+
+    /// unescape "~1" to tilde and "~0" to slash (order is important!)
+    static void unescape(std::string& s)
+    {
+        replace_substring(s, "~1", "/");
+        replace_substring(s, "~0", "~");
+    }
+
+    /*!
+    @param[in] reference_string  the reference string to the current value
+    @param[in] value             the value to consider
+    @param[in,out] result        the result object to insert values to
+
+    @note Empty objects or arrays are flattened to `null`.
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    static void flatten(const std::string& reference_string,
+                        const NLOHMANN_BASIC_JSON_TPL& value,
+                        NLOHMANN_BASIC_JSON_TPL& result);
+
+    /*!
+    @param[in] value  flattened JSON
+
+    @return unflattened JSON
+
+    @throw parse_error.109 if array index is not a number
+    @throw type_error.314  if value is not an object
+    @throw type_error.315  if object values are not primitive
+    @throw type_error.313  if value cannot be unflattened
+    */
+    NLOHMANN_BASIC_JSON_TPL_DECLARATION
+    static NLOHMANN_BASIC_JSON_TPL
+    unflatten(const NLOHMANN_BASIC_JSON_TPL& value);
+
+    friend bool operator==(json_pointer const& lhs,
+                           json_pointer const& rhs) noexcept;
+
+    friend bool operator!=(json_pointer const& lhs,
+                           json_pointer const& rhs) noexcept;
+
+    /// the reference tokens
+    std::vector<std::string> reference_tokens;
+};
 
 /*!
 @brief a class to store JSON values
@@ -1370,35 +7319,53 @@ Format](http://rfc7159.net/rfc7159)
 
 @nosubgrouping
 */
-template <
-    template<typename U, typename V, typename... Args> class ObjectType = std::map,
-    template<typename U, typename... Args> class ArrayType = std::vector,
-    class StringType = std::string,
-    class BooleanType = bool,
-    class NumberIntegerType = std::int64_t,
-    class NumberUnsignedType = std::uint64_t,
-    class NumberFloatType = double,
-    template<typename U> class AllocatorType = std::allocator,
-    template<typename T, typename SFINAE = void> class JSONSerializer = adl_serializer
-    >
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
 class basic_json
 {
   private:
     template<detail::value_t> friend struct detail::external_constructor;
+    friend ::nlohmann::json_pointer;
+    friend ::nlohmann::detail::parser<basic_json>;
+    friend ::nlohmann::detail::serializer<basic_json>;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::iter_impl;
+    template<typename BasicJsonType, typename CharType>
+    friend class ::nlohmann::detail::binary_writer;
+    template<typename BasicJsonType>
+    friend class ::nlohmann::detail::binary_reader;
+
     /// workaround type for MSVC
-    using basic_json_t = basic_json<ObjectType, ArrayType, StringType,
-          BooleanType, NumberIntegerType, NumberUnsignedType, NumberFloatType,
-          AllocatorType, JSONSerializer>;
+    using basic_json_t = NLOHMANN_BASIC_JSON_TPL;
+
+    // convenience aliases for types residing in namespace detail;
+    using lexer = ::nlohmann::detail::lexer<basic_json>;
+    using parser = ::nlohmann::detail::parser<basic_json>;
+
+    using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t;
+    template<typename BasicJsonType>
+    using internal_iterator = ::nlohmann::detail::internal_iterator<BasicJsonType>;
+    template<typename BasicJsonType>
+    using iter_impl = ::nlohmann::detail::iter_impl<BasicJsonType>;
+    template<typename Iterator>
+    using iteration_proxy = ::nlohmann::detail::iteration_proxy<Iterator>;
+    template<typename Base> using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator<Base>;
+
+    template<typename CharType>
+    using output_adapter_t = ::nlohmann::detail::output_adapter_t<CharType>;
+
+    using binary_reader = ::nlohmann::detail::binary_reader<basic_json>;
+    template<typename CharType> using binary_writer = ::nlohmann::detail::binary_writer<basic_json, CharType>;
+
+    using serializer = ::nlohmann::detail::serializer<basic_json>;
 
   public:
     using value_t = detail::value_t;
     // forward declarations
-    template<typename U> class iter_impl;
-    template<typename Base> class json_reverse_iterator;
-    class json_pointer;
+    using json_pointer = ::nlohmann::json_pointer;
     template<typename T, typename SFINAE>
     using json_serializer = JSONSerializer<T, SFINAE>;
 
+    using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>;
 
     ////////////////
     // exceptions //
@@ -1493,6 +7460,9 @@ class basic_json
     @liveexample{The following code shows an example output of the `meta()`
     function.,meta}
 
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
     @complexity Constant.
 
     @since 2.1.0
@@ -1521,10 +7491,10 @@ class basic_json
         result["platform"] = "unknown";
 #endif
 
-#if defined(__clang__)
-        result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}};
-#elif defined(__ICC) || defined(__INTEL_COMPILER)
+#if defined(__ICC) || defined(__INTEL_COMPILER)
         result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}};
+#elif defined(__clang__)
+        result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}};
 #elif defined(__GNUC__) || defined(__GNUG__)
         result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}};
 #elif defined(__HP_cc) || defined(__HP_aCC)
@@ -1619,7 +7589,7 @@ class basic_json
     [RFC 7159](http://rfc7159.net/rfc7159) specifies:
     > An implementation may set limits on the maximum depth of nesting.
 
-    In this class, the object's limit of nesting is not constraint explicitly.
+    In this class, the object's limit of nesting is not explicitly constrained.
     However, a maximum depth of nesting may be introduced by the compiler or
     runtime environment. A theoretical limit can be queried by calling the
     @ref max_size function of a JSON object.
@@ -1678,7 +7648,7 @@ class basic_json
     [RFC 7159](http://rfc7159.net/rfc7159) specifies:
     > An implementation may set limits on the maximum depth of nesting.
 
-    In this class, the array's limit of nesting is not constraint explicitly.
+    In this class, the array's limit of nesting is not explicitly constrained.
     However, a maximum depth of nesting may be introduced by the compiler or
     runtime environment. A theoretical limit can be queried by calling the
     @ref max_size function of a JSON array.
@@ -2127,17 +8097,70 @@ class basic_json
             string = create<string_t>(value);
         }
 
+        /// constructor for rvalue strings
+        json_value(string_t&& value)
+        {
+            string = create<string_t>(std::move(value));
+        }
+
         /// constructor for objects
         json_value(const object_t& value)
         {
             object = create<object_t>(value);
         }
 
+        /// constructor for rvalue objects
+        json_value(object_t&& value)
+        {
+            object = create<object_t>(std::move(value));
+        }
+
         /// constructor for arrays
         json_value(const array_t& value)
         {
             array = create<array_t>(value);
         }
+
+        /// constructor for rvalue arrays
+        json_value(array_t&& value)
+        {
+            array = create<array_t>(std::move(value));
+        }
+
+        void destroy(value_t t)
+        {
+            switch (t)
+            {
+                case value_t::object:
+                {
+                    AllocatorType<object_t> alloc;
+                    alloc.destroy(object);
+                    alloc.deallocate(object, 1);
+                    break;
+                }
+
+                case value_t::array:
+                {
+                    AllocatorType<array_t> alloc;
+                    alloc.destroy(array);
+                    alloc.deallocate(array, 1);
+                    break;
+                }
+
+                case value_t::string:
+                {
+                    AllocatorType<string_t> alloc;
+                    alloc.destroy(string);
+                    alloc.deallocate(string, 1);
+                    break;
+                }
+
+                default:
+                {
+                    break;
+                }
+            }
+        }
     };
 
     /*!
@@ -2161,31 +8184,7 @@ class basic_json
     // JSON parser callback //
     //////////////////////////
 
-    /*!
-    @brief JSON callback events
-
-    This enumeration lists the parser events that can trigger calling a
-    callback function of type @ref parser_callback_t during parsing.
-
-    @image html callback_events.png "Example when certain parse events are triggered"
-
-    @since version 1.0.0
-    */
-    enum class parse_event_t : uint8_t
-    {
-        /// the parser read `{` and started to process a JSON object
-        object_start,
-        /// the parser read `}` and finished processing a JSON object
-        object_end,
-        /// the parser read `[` and started to process a JSON array
-        array_start,
-        /// the parser read `]` and finished processing a JSON array
-        array_end,
-        /// the parser read a key of a value in an object
-        key,
-        /// the parser finished reading a JSON value
-        value
-    };
+    using parse_event_t = typename parser::parse_event_t;
 
     /*!
     @brief per-element parser callback type
@@ -2239,9 +8238,7 @@ class basic_json
 
     @since version 1.0.0
     */
-    using parser_callback_t = std::function<bool(int depth,
-                              parse_event_t event,
-                              basic_json& parsed)>;
+    using parser_callback_t = typename parser::parser_callback_t;
 
 
     //////////////////
@@ -2268,17 +8265,22 @@ class basic_json
     object      | `{}`
     array       | `[]`
 
-    @param[in] value_type  the type of the value to create
+    @param[in] v  the type of the value to create
 
     @complexity Constant.
 
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
     @liveexample{The following code shows the constructor for different @ref
     value_t values,basic_json__value_t}
 
+    @sa @ref clear() -- restores the postcondition of this constructor
+
     @since version 1.0.0
     */
-    basic_json(const value_t value_type)
-        : m_type(value_type), m_value(value_type)
+    basic_json(const value_t v)
+        : m_type(v), m_value(v)
     {
         assert_invariant();
     }
@@ -2319,9 +8321,9 @@ class basic_json
     following types:
     - **arrays**: @ref array_t and all kinds of compatible containers such as
       `std::vector`, `std::deque`, `std::list`, `std::forward_list`,
-      `std::array`, `std::set`, `std::unordered_set`, `std::multiset`, and
-      `unordered_multiset` with a `value_type` from which a @ref basic_json
-      value can be constructed.
+      `std::array`, `std::valarray`, `std::set`, `std::unordered_set`,
+      `std::multiset`, and `std::unordered_multiset` with a `value_type` from
+      which a @ref basic_json value can be constructed.
     - **objects**: @ref object_t and all kinds of compatible associative
       containers such as `std::map`, `std::unordered_map`, `std::multimap`,
       and `std::unordered_multimap` with a `key_type` compatible to
@@ -2347,13 +8349,16 @@ class basic_json
 
     @tparam U = `uncvref_t<CompatibleType>`
 
-    @param[in] val the value to be forwarded
+    @param[in] val the value to be forwarded to the respective constructor
 
     @complexity Usually linear in the size of the passed @a val, also
                 depending on the implementation of the called `to_json()`
                 method.
 
-    @throw what `json_serializer<U>::to_json()` throws
+    @exceptionsafety Depends on the called constructor. For types directly
+    supported by the library (i.e., all types for which no `to_json()` function
+    was provided), strong guarantee holds: if an exception is thrown, there are
+    no changes to any JSON value.
 
     @liveexample{The following code shows the constructor with several
     compatible types.,basic_json__CompatibleType}
@@ -2393,7 +8398,7 @@ class basic_json
 
     1. The empty initializer list is written as `{}` which is exactly an empty
        JSON object.
-    2. C++ has now way of describing mapped types other than to list a list of
+    2. C++ has no way of describing mapped types other than to list a list of
        pairs. As JSON requires that keys must be of type string, rule 2 is the
        weakest constraint one can pose on initializer lists to interpret them
        as an object.
@@ -2403,10 +8408,10 @@ class basic_json
     With the rules described above, the following JSON values cannot be
     expressed by an initializer list:
 
-    - the empty array (`[]`): use @ref array(std::initializer_list<basic_json>)
+    - the empty array (`[]`): use @ref array(initializer_list_t)
       with an empty initializer list in this case
     - arrays whose elements satisfy rule 2: use @ref
-      array(std::initializer_list<basic_json>) with the same initializer list
+      array(initializer_list_t) with the same initializer list
       in this case
 
     @note When used without parentheses around an empty initializer list, @ref
@@ -2418,8 +8423,8 @@ class basic_json
     @param[in] type_deduction internal parameter; when set to `true`, the type
     of the JSON value is deducted from the initializer list @a init; when set
     to `false`, the type provided via @a manual_type is forced. This mode is
-    used by the functions @ref array(std::initializer_list<basic_json>) and
-    @ref object(std::initializer_list<basic_json>).
+    used by the functions @ref array(initializer_list_t) and
+    @ref object(initializer_list_t).
 
     @param[in] manual_type internal parameter; when @a type_deduction is set
     to `false`, the created JSON value will use the provided type (only @ref
@@ -2430,31 +8435,34 @@ class basic_json
     `value_t::object`, but @a init contains an element which is not a pair
     whose first element is a string. In this case, the constructor could not
     create an object. If @a type_deduction would have be `true`, an array
-    would have been created. See @ref object(std::initializer_list<basic_json>)
+    would have been created. See @ref object(initializer_list_t)
     for an example.
 
     @complexity Linear in the size of the initializer list @a init.
 
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
     @liveexample{The example below shows how JSON values are created from
     initializer lists.,basic_json__list_init_t}
 
-    @sa @ref array(std::initializer_list<basic_json>) -- create a JSON array
+    @sa @ref array(initializer_list_t) -- create a JSON array
     value from an initializer list
-    @sa @ref object(std::initializer_list<basic_json>) -- create a JSON object
+    @sa @ref object(initializer_list_t) -- create a JSON object
     value from an initializer list
 
     @since version 1.0.0
     */
-    basic_json(std::initializer_list<basic_json> init,
+    basic_json(initializer_list_t init,
                bool type_deduction = true,
                value_t manual_type = value_t::array)
     {
         // check if each element is an array with two elements whose first
         // element is a string
         bool is_an_object = std::all_of(init.begin(), init.end(),
-                                        [](const basic_json & element)
+                                        [](const detail::json_ref<basic_json>& element_ref)
         {
-            return element.is_array() and element.size() == 2 and element[0].is_string();
+            return (element_ref->is_array() and element_ref->size() == 2 and (*element_ref)[0].is_string());
         });
 
         // adjust type if type deduction is not wanted
@@ -2467,7 +8475,7 @@ class basic_json
             }
 
             // if object is wanted but impossible, throw an exception
-            if (manual_type == value_t::object and not is_an_object)
+            if (JSON_UNLIKELY(manual_type == value_t::object and not is_an_object))
             {
                 JSON_THROW(type_error::create(301, "cannot create object from initializer list"));
             }
@@ -2479,16 +8487,19 @@ class basic_json
             m_type = value_t::object;
             m_value = value_t::object;
 
-            std::for_each(init.begin(), init.end(), [this](const basic_json & element)
+            std::for_each(init.begin(), init.end(), [this](const detail::json_ref<basic_json>& element_ref)
             {
-                m_value.object->emplace(*(element[0].m_value.string), element[1]);
+                auto element = element_ref.moved_or_copied();
+                m_value.object->emplace(
+                    std::move(*((*element.m_value.array)[0].m_value.string)),
+                    std::move((*element.m_value.array)[1]));
             });
         }
         else
         {
             // the initializer list describes an array -> create array
             m_type = value_t::array;
-            m_value.array = create<array_t>(init);
+            m_value.array = create<array_t>(init.begin(), init.end());
         }
 
         assert_invariant();
@@ -2503,7 +8514,7 @@ class basic_json
 
     @note This function is only needed to express two edge cases that cannot
     be realized with the initializer list constructor (@ref
-    basic_json(std::initializer_list<basic_json>, bool, value_t)). These cases
+    basic_json(initializer_list_t, bool, value_t)). These cases
     are:
     1. creating an array whose elements are all pairs whose first element is a
     string -- in this case, the initializer list constructor would create an
@@ -2518,18 +8529,20 @@ class basic_json
 
     @complexity Linear in the size of @a init.
 
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
     @liveexample{The following code shows an example for the `array`
     function.,array}
 
-    @sa @ref basic_json(std::initializer_list<basic_json>, bool, value_t) --
+    @sa @ref basic_json(initializer_list_t, bool, value_t) --
     create a JSON value from an initializer list
-    @sa @ref object(std::initializer_list<basic_json>) -- create a JSON object
+    @sa @ref object(initializer_list_t) -- create a JSON object
     value from an initializer list
 
     @since version 1.0.0
     */
-    static basic_json array(std::initializer_list<basic_json> init =
-                                std::initializer_list<basic_json>())
+    static basic_json array(initializer_list_t init = {})
     {
         return basic_json(init, false, value_t::array);
     }
@@ -2542,10 +8555,10 @@ class basic_json
     the initializer list is empty, the empty object `{}` is created.
 
     @note This function is only added for symmetry reasons. In contrast to the
-    related function @ref array(std::initializer_list<basic_json>), there are
+    related function @ref array(initializer_list_t), there are
     no cases which can only be expressed by this function. That is, any
     initializer list @a init can also be passed to the initializer list
-    constructor @ref basic_json(std::initializer_list<basic_json>, bool, value_t).
+    constructor @ref basic_json(initializer_list_t, bool, value_t).
 
     @param[in] init  initializer list to create an object from (optional)
 
@@ -2553,24 +8566,26 @@ class basic_json
 
     @throw type_error.301 if @a init is not a list of pairs whose first
     elements are strings. In this case, no object can be created. When such a
-    value is passed to @ref basic_json(std::initializer_list<basic_json>, bool, value_t),
+    value is passed to @ref basic_json(initializer_list_t, bool, value_t),
     an array would have been created from the passed initializer list @a init.
     See example below.
 
     @complexity Linear in the size of @a init.
 
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
     @liveexample{The following code shows an example for the `object`
     function.,object}
 
-    @sa @ref basic_json(std::initializer_list<basic_json>, bool, value_t) --
+    @sa @ref basic_json(initializer_list_t, bool, value_t) --
     create a JSON value from an initializer list
-    @sa @ref array(std::initializer_list<basic_json>) -- create a JSON array
+    @sa @ref array(initializer_list_t) -- create a JSON array
     value from an initializer list
 
     @since version 1.0.0
     */
-    static basic_json object(std::initializer_list<basic_json> init =
-                                 std::initializer_list<basic_json>())
+    static basic_json object(initializer_list_t init = {})
     {
         return basic_json(init, false, value_t::object);
     }
@@ -2579,14 +8594,18 @@ class basic_json
     @brief construct an array with count copies of given value
 
     Constructs a JSON array value by creating @a cnt copies of a passed value.
-    In case @a cnt is `0`, an empty array is created. As postcondition,
-    `std::distance(begin(),end()) == cnt` holds.
+    In case @a cnt is `0`, an empty array is created.
 
     @param[in] cnt  the number of JSON copies of @a val to create
     @param[in] val  the JSON value to copy
 
+    @post `std::distance(begin(),end()) == cnt` holds.
+
     @complexity Linear in @a cnt.
 
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
     @liveexample{The following code shows examples for the @ref
     basic_json(size_type\, const basic_json&)
     constructor.,basic_json__size_type_basic_json}
@@ -2605,12 +8624,13 @@ class basic_json
 
     Constructs the JSON value with the contents of the range `[first, last)`.
     The semantics depends on the different types a JSON value can have:
-    - In case of primitive types (number, boolean, or string), @a first must
-      be `begin()` and @a last must be `end()`. In this case, the value is
+    - In case of a null type, invalid_iterator.206 is thrown.
+    - In case of other primitive types (number, boolean, or string), @a first
+      must be `begin()` and @a last must be `end()`. In this case, the value is
       copied. Otherwise, invalid_iterator.204 is thrown.
     - In case of structured types (array, object), the constructor behaves as
-      similar versions for `std::vector`.
-    - In case of a null type, invalid_iterator.206 is thrown.
+      similar versions for `std::vector` or `std::map`; that is, a JSON array
+      or object is constructed from the values in the range.
 
     @tparam InputIT an input iterator type (@ref iterator or @ref
     const_iterator)
@@ -2619,11 +8639,20 @@ class basic_json
     @param[in] last end of the range to copy from (excluded)
 
     @pre Iterators @a first and @a last must be initialized. **This
-         precondition is enforced with an assertion.**
+         precondition is enforced with an assertion (see warning).** If
+         assertions are switched off, a violation of this precondition yields
+         undefined behavior.
 
     @pre Range `[first, last)` is valid. Usually, this precondition cannot be
          checked efficiently. Only certain edge cases are detected; see the
-         description of the exceptions below.
+         description of the exceptions below. A violation of this precondition
+         yields undefined behavior.
+
+    @warning A precondition is enforced with a runtime assertion that will
+             result in calling `std::abort` if this precondition is not met.
+             Assertions can be disabled by defining `NDEBUG` at compile time.
+             See http://en.cppreference.com/w/cpp/error/assert for more
+             information.
 
     @throw invalid_iterator.201 if iterators @a first and @a last are not
     compatible (i.e., do not belong to the same JSON value). In this case,
@@ -2637,6 +8666,9 @@ class basic_json
 
     @complexity Linear in distance between @a first and @a last.
 
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
     @liveexample{The example below shows several ways to create JSON values by
     specifying a subrange with iterators.,basic_json__InputIt_InputIt}
 
@@ -2651,7 +8683,7 @@ class basic_json
         assert(last.m_object != nullptr);
 
         // make sure iterator fits the current value
-        if (first.m_object != last.m_object)
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
         {
             JSON_THROW(invalid_iterator::create(201, "iterators are not compatible"));
         }
@@ -2668,7 +8700,8 @@ class basic_json
             case value_t::number_unsigned:
             case value_t::string:
             {
-                if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end())
+                if (JSON_UNLIKELY(not first.m_it.primitive_iterator.is_begin()
+                                  or not last.m_it.primitive_iterator.is_end()))
                 {
                     JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
                 }
@@ -2676,9 +8709,7 @@ class basic_json
             }
 
             default:
-            {
                 break;
-            }
         }
 
         switch (m_type)
@@ -2728,10 +8759,8 @@ class basic_json
             }
 
             default:
-            {
                 JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " +
-                                                    first.m_object->type_name()));
-            }
+                                                    std::string(first.m_object->type_name())));
         }
 
         assert_invariant();
@@ -2742,6 +8771,11 @@ class basic_json
     // other constructors and destructor //
     ///////////////////////////////////////
 
+    /// @private
+    basic_json(const detail::json_ref<basic_json>& ref)
+        : basic_json(ref.moved_or_copied())
+    {}
+
     /*!
     @brief copy constructor
 
@@ -2749,8 +8783,13 @@ class basic_json
 
     @param[in] other  the JSON value to copy
 
+    @post `*this == other`
+
     @complexity Linear in the size of @a other.
 
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes to any JSON value.
+
     @requirement This function helps `basic_json` satisfying the
     [Container](http://en.cppreference.com/w/cpp/concept/Container)
     requirements:
@@ -2813,9 +8852,7 @@ class basic_json
             }
 
             default:
-            {
                 break;
-            }
         }
 
         assert_invariant();
@@ -2830,10 +8867,18 @@ class basic_json
 
     @param[in,out] other  value to move to this object
 
-    @post @a other is a JSON null value
+    @post `*this` has the same value as @a other before the call.
+    @post @a other is a JSON null value.
 
     @complexity Constant.
 
+    @exceptionsafety No-throw guarantee: this constructor never throws
+    exceptions.
+
+    @requirement This function helps `basic_json` satisfying the
+    [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible)
+    requirements.
+
     @liveexample{The code below shows the move constructor explicitly called
     via std::move.,basic_json__moveconstructor}
 
@@ -2858,7 +8903,7 @@ class basic_json
 
     Copy assignment operator. Copies a JSON value via the "copy and swap"
     strategy: It is expressed in terms of the copy constructor, destructor,
-    and the swap() member function.
+    and the `swap()` member function.
 
     @param[in] other  value to copy from
 
@@ -2912,39 +8957,7 @@ class basic_json
     ~basic_json()
     {
         assert_invariant();
-
-        switch (m_type)
-        {
-            case value_t::object:
-            {
-                AllocatorType<object_t> alloc;
-                alloc.destroy(m_value.object);
-                alloc.deallocate(m_value.object, 1);
-                break;
-            }
-
-            case value_t::array:
-            {
-                AllocatorType<array_t> alloc;
-                alloc.destroy(m_value.array);
-                alloc.deallocate(m_value.array, 1);
-                break;
-            }
-
-            case value_t::string:
-            {
-                AllocatorType<string_t> alloc;
-                alloc.destroy(m_value.string);
-                alloc.deallocate(m_value.string, 1);
-                break;
-            }
-
-            default:
-            {
-                // all other types need no specific destructor
-                break;
-            }
-        }
+        m_value.destroy(m_type);
     }
 
     /// @}
@@ -2963,38 +8976,47 @@ class basic_json
 
     Serialization function for JSON values. The function tries to mimic
     Python's `json.dumps()` function, and currently supports its @a indent
-    parameter.
+    and @a ensure_ascii parameters.
 
     @param[in] indent If indent is nonnegative, then array elements and object
     members will be pretty-printed with that indent level. An indent level of
     `0` will only insert newlines. `-1` (the default) selects the most compact
     representation.
-    @param[in] indent_char The character to use for indentation of @a indent is
-    greate than `0`. The default is ` ` (space).
+    @param[in] indent_char The character to use for indentation if @a indent is
+    greater than `0`. The default is ` ` (space).
+    @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
+    in the output are escaped with \uXXXX sequences, and the result consists
+    of ASCII characters only.
 
     @return string containing the serialization of the JSON value
 
     @complexity Linear.
 
-    @liveexample{The following example shows the effect of different @a indent
-    parameters to the result of the serialization.,dump}
+    @exceptionsafety Strong guarantee: if an exception is thrown, there are no
+    changes in the JSON value.
+
+    @liveexample{The following example shows the effect of different @a indent\,
+    @a indent_char\, and @a ensure_ascii parameters to the result of the
+    serialization.,dump}
 
     @see https://docs.python.org/2/library/json.html#json.dump
 
-    @since version 1.0.0; indentaction character added in version 3.0.0
+    @since version 1.0.0; indentation character @a indent_char and option
+           @a ensure_ascii added in version 3.0.0
     */
-    string_t dump(const int indent = -1, const char indent_char = ' ') const
+    string_t dump(const int indent = -1, const char indent_char = ' ',
+                  const bool ensure_ascii = false) const
     {
         string_t result;
-        serializer s(output_adapter<char>::create(result), indent_char);
+        serializer s(detail::output_adapter<char>(result), indent_char);
 
         if (indent >= 0)
         {
-            s.dump(*this, true, static_cast<unsigned int>(indent));
+            s.dump(*this, true, ensure_ascii, static_cast<unsigned int>(indent));
         }
         else
         {
-            s.dump(*this, false, 0);
+            s.dump(*this, false, ensure_ascii, 0);
         }
 
         return result;
@@ -3007,6 +9029,17 @@ class basic_json
     enumeration.
 
     @return the type of the JSON value
+            Value type                | return value
+            ------------------------- | -------------------------
+            null                      | value_t::null
+            boolean                   | value_t::boolean
+            string                    | value_t::string
+            number (integer)          | value_t::number_integer
+            number (unsigned integer) | value_t::number_unsigned
+            number (foating-point)    | value_t::number_float
+            object                    | value_t::object
+            array                     | value_t::array
+            discarded                 | value_t::discarded
 
     @complexity Constant.
 
@@ -3016,6 +9049,9 @@ class basic_json
     @liveexample{The following code exemplifies `type()` for all JSON
     types.,type}
 
+    @sa @ref operator value_t() -- return the type of the JSON value (implicit) 
+    @sa @ref type_name() -- return the type as string
+
     @since version 1.0.0
     */
     constexpr value_t type() const noexcept
@@ -3026,8 +9062,8 @@ class basic_json
     /*!
     @brief return whether type is primitive
 
-    This function returns true iff the JSON type is primitive (string, number,
-    boolean, or null).
+    This function returns true if and only if the JSON type is primitive
+    (string, number, boolean, or null).
 
     @return `true` if type is primitive (string, number, boolean, or null),
     `false` otherwise.
@@ -3056,8 +9092,8 @@ class basic_json
     /*!
     @brief return whether type is structured
 
-    This function returns true iff the JSON type is structured (array or
-    object).
+    This function returns true if and only if the JSON type is structured
+    (array or object).
 
     @return `true` if type is structured (array or object), `false` otherwise.
 
@@ -3083,7 +9119,7 @@ class basic_json
     /*!
     @brief return whether value is null
 
-    This function returns true iff the JSON value is null.
+    This function returns true if and only if the JSON value is null.
 
     @return `true` if type is null, `false` otherwise.
 
@@ -3099,13 +9135,13 @@ class basic_json
     */
     constexpr bool is_null() const noexcept
     {
-        return m_type == value_t::null;
+        return (m_type == value_t::null);
     }
 
     /*!
     @brief return whether value is a boolean
 
-    This function returns true iff the JSON value is a boolean.
+    This function returns true if and only if the JSON value is a boolean.
 
     @return `true` if type is boolean, `false` otherwise.
 
@@ -3121,14 +9157,14 @@ class basic_json
     */
     constexpr bool is_boolean() const noexcept
     {
-        return m_type == value_t::boolean;
+        return (m_type == value_t::boolean);
     }
 
     /*!
     @brief return whether value is a number
 
-    This function returns true iff the JSON value is a number. This includes
-    both integer and floating-point values.
+    This function returns true if and only if the JSON value is a number. This
+    includes both integer (signed and unsigned) and floating-point values.
 
     @return `true` if type is number (regardless whether integer, unsigned
     integer or floating-type), `false` otherwise.
@@ -3157,8 +9193,8 @@ class basic_json
     /*!
     @brief return whether value is an integer number
 
-    This function returns true iff the JSON value is an integer or unsigned
-    integer number. This excludes floating-point values.
+    This function returns true if and only if the JSON value is a signed or
+    unsigned integer number. This excludes floating-point values.
 
     @return `true` if type is an integer or unsigned integer number, `false`
     otherwise.
@@ -3180,14 +9216,14 @@ class basic_json
     */
     constexpr bool is_number_integer() const noexcept
     {
-        return m_type == value_t::number_integer or m_type == value_t::number_unsigned;
+        return (m_type == value_t::number_integer or m_type == value_t::number_unsigned);
     }
 
     /*!
     @brief return whether value is an unsigned integer number
 
-    This function returns true iff the JSON value is an unsigned integer
-    number. This excludes floating-point and (signed) integer values.
+    This function returns true if and only if the JSON value is an unsigned
+    integer number. This excludes floating-point and signed integer values.
 
     @return `true` if type is an unsigned integer number, `false` otherwise.
 
@@ -3208,14 +9244,14 @@ class basic_json
     */
     constexpr bool is_number_unsigned() const noexcept
     {
-        return m_type == value_t::number_unsigned;
+        return (m_type == value_t::number_unsigned);
     }
 
     /*!
     @brief return whether value is a floating-point number
 
-    This function returns true iff the JSON value is a floating-point number.
-    This excludes integer and unsigned integer values.
+    This function returns true if and only if the JSON value is a
+    floating-point number. This excludes signed and unsigned integer values.
 
     @return `true` if type is a floating-point number, `false` otherwise.
 
@@ -3236,13 +9272,13 @@ class basic_json
     */
     constexpr bool is_number_float() const noexcept
     {
-        return m_type == value_t::number_float;
+        return (m_type == value_t::number_float);
     }
 
     /*!
     @brief return whether value is an object
 
-    This function returns true iff the JSON value is an object.
+    This function returns true if and only if the JSON value is an object.
 
     @return `true` if type is object, `false` otherwise.
 
@@ -3258,13 +9294,13 @@ class basic_json
     */
     constexpr bool is_object() const noexcept
     {
-        return m_type == value_t::object;
+        return (m_type == value_t::object);
     }
 
     /*!
     @brief return whether value is an array
 
-    This function returns true iff the JSON value is an array.
+    This function returns true if and only if the JSON value is an array.
 
     @return `true` if type is array, `false` otherwise.
 
@@ -3280,13 +9316,13 @@ class basic_json
     */
     constexpr bool is_array() const noexcept
     {
-        return m_type == value_t::array;
+        return (m_type == value_t::array);
     }
 
     /*!
     @brief return whether value is a string
 
-    This function returns true iff the JSON value is a string.
+    This function returns true if and only if the JSON value is a string.
 
     @return `true` if type is string, `false` otherwise.
 
@@ -3302,14 +9338,14 @@ class basic_json
     */
     constexpr bool is_string() const noexcept
     {
-        return m_type == value_t::string;
+        return (m_type == value_t::string);
     }
 
     /*!
     @brief return whether value is discarded
 
-    This function returns true iff the JSON value was discarded during parsing
-    with a callback function (see @ref parser_callback_t).
+    This function returns true if and only if the JSON value was discarded
+    during parsing with a callback function (see @ref parser_callback_t).
 
     @note This function will always be `false` for JSON values after parsing.
     That is, discarded values can only occur during parsing, but will be
@@ -3329,7 +9365,7 @@ class basic_json
     */
     constexpr bool is_discarded() const noexcept
     {
-        return m_type == value_t::discarded;
+        return (m_type == value_t::discarded);
     }
 
     /*!
@@ -3348,6 +9384,9 @@ class basic_json
     @liveexample{The following code exemplifies the @ref value_t operator for
     all JSON types.,operator__value_t}
 
+    @sa @ref type() -- return the type of the JSON value (explicit)
+    @sa @ref type_name() -- return the type as string
+
     @since version 1.0.0
     */
     constexpr operator value_t() const noexcept
@@ -3365,12 +9404,12 @@ class basic_json
     /// get a boolean (explicit)
     boolean_t get_impl(boolean_t* /*unused*/) const
     {
-        if (is_boolean())
+        if (JSON_LIKELY(is_boolean()))
         {
             return m_value.boolean;
         }
 
-        JSON_THROW(type_error::create(302, "type must be boolean, but is " + type_name()));
+        JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(type_name())));
     }
 
     /// get a pointer to the value (object)
@@ -3460,7 +9499,7 @@ class basic_json
     /*!
     @brief helper function to implement get_ref()
 
-    This funcion helps to implement get_ref() without code duplication for
+    This function helps to implement get_ref() without code duplication for
     const and non-const overloads
 
     @tparam ThisType will be deduced as `basic_json` or `const basic_json`
@@ -3471,18 +9510,15 @@ class basic_json
     template<typename ReferenceType, typename ThisType>
     static ReferenceType get_ref_impl(ThisType& obj)
     {
-        // helper type
-        using PointerType = typename std::add_pointer<ReferenceType>::type;
-
         // delegate the call to get_ptr<>()
-        auto ptr = obj.template get_ptr<PointerType>();
+        auto ptr = obj.template get_ptr<typename std::add_pointer<ReferenceType>::type>();
 
-        if (ptr != nullptr)
+        if (JSON_LIKELY(ptr != nullptr))
         {
             return *ptr;
         }
 
-        JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + obj.type_name()));
+        JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + std::string(obj.type_name())));
     }
 
   public:
@@ -3824,6 +9860,7 @@ class basic_json
     */
     template < typename ValueType, typename std::enable_if <
                    not std::is_pointer<ValueType>::value and
+                   not std::is_same<ValueType, detail::json_ref<basic_json>>::value and
                    not std::is_same<ValueType, typename string_t::value_type>::value
 #ifndef _MSC_VER  // fix for issue #167 operator<< ambiguity under VS2015
                    and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value
@@ -3878,7 +9915,7 @@ class basic_json
     reference at(size_type idx)
     {
         // at only works for arrays
-        if (is_array())
+        if (JSON_LIKELY(is_array()))
         {
             JSON_TRY
             {
@@ -3892,7 +9929,7 @@ class basic_json
         }
         else
         {
-            JSON_THROW(type_error::create(304, "cannot use at() with " + type_name()));
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
         }
     }
 
@@ -3925,7 +9962,7 @@ class basic_json
     const_reference at(size_type idx) const
     {
         // at only works for arrays
-        if (is_array())
+        if (JSON_LIKELY(is_array()))
         {
             JSON_TRY
             {
@@ -3939,7 +9976,7 @@ class basic_json
         }
         else
         {
-            JSON_THROW(type_error::create(304, "cannot use at() with " + type_name()));
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
         }
     }
 
@@ -3976,7 +10013,7 @@ class basic_json
     reference at(const typename object_t::key_type& key)
     {
         // at only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             JSON_TRY
             {
@@ -3990,7 +10027,7 @@ class basic_json
         }
         else
         {
-            JSON_THROW(type_error::create(304, "cannot use at() with " + type_name()));
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
         }
     }
 
@@ -4027,7 +10064,7 @@ class basic_json
     const_reference at(const typename object_t::key_type& key) const
     {
         // at only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             JSON_TRY
             {
@@ -4041,7 +10078,7 @@ class basic_json
         }
         else
         {
-            JSON_THROW(type_error::create(304, "cannot use at() with " + type_name()));
+            JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name())));
         }
     }
 
@@ -4081,7 +10118,7 @@ class basic_json
         }
 
         // operator[] only works for arrays
-        if (is_array())
+        if (JSON_LIKELY(is_array()))
         {
             // fill up array with null values if given idx is outside range
             if (idx >= m_value.array->size())
@@ -4094,7 +10131,7 @@ class basic_json
             return m_value.array->operator[](idx);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
     }
 
     /*!
@@ -4119,12 +10156,12 @@ class basic_json
     const_reference operator[](size_type idx) const
     {
         // const operator[] only works for arrays
-        if (is_array())
+        if (JSON_LIKELY(is_array()))
         {
             return m_value.array->operator[](idx);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
     }
 
     /*!
@@ -4165,12 +10202,12 @@ class basic_json
         }
 
         // operator[] only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             return m_value.object->operator[](key);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
     }
 
     /*!
@@ -4206,81 +10243,13 @@ class basic_json
     const_reference operator[](const typename object_t::key_type& key) const
     {
         // const operator[] only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             assert(m_value.object->find(key) != m_value.object->end());
             return m_value.object->find(key)->second;
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
-    }
-
-    /*!
-    @brief access specified object element
-
-    Returns a reference to the element at with specified key @a key.
-
-    @note If @a key is not found in the object, then it is silently added to
-    the object and filled with a `null` value to make `key` a valid reference.
-    In case the value was `null` before, it is converted to an object.
-
-    @param[in] key  key of the element to access
-
-    @return reference to the element at key @a key
-
-    @throw type_error.305 if the JSON value is not an object or null; in that
-    cases, using the [] operator with a key makes no sense.
-
-    @complexity Logarithmic in the size of the container.
-
-    @liveexample{The example below shows how object elements can be read and
-    written using the `[]` operator.,operatorarray__key_type}
-
-    @sa @ref at(const typename object_t::key_type&) for access by reference
-    with range checking
-    @sa @ref value() for access by value with a default value
-
-    @since version 1.0.0
-    */
-    template<typename T, std::size_t n>
-    reference operator[](T * (&key)[n])
-    {
-        return operator[](static_cast<const T>(key));
-    }
-
-    /*!
-    @brief read-only access specified object element
-
-    Returns a const reference to the element at with specified key @a key. No
-    bounds checking is performed.
-
-    @warning If the element with key @a key does not exist, the behavior is
-    undefined.
-
-    @note This function is required for compatibility reasons with Clang.
-
-    @param[in] key  key of the element to access
-
-    @return const reference to the element at key @a key
-
-    @throw type_error.305 if the JSON value is not an object; in that cases,
-    using the [] operator with a key makes no sense.
-
-    @complexity Logarithmic in the size of the container.
-
-    @liveexample{The example below shows how object elements can be read using
-    the `[]` operator.,operatorarray__key_type_const}
-
-    @sa @ref at(const typename object_t::key_type&) for access by reference
-    with range checking
-    @sa @ref value() for access by value with a default value
-
-    @since version 1.0.0
-    */
-    template<typename T, std::size_t n>
-    const_reference operator[](T * (&key)[n]) const
-    {
-        return operator[](static_cast<const T>(key));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
     }
 
     /*!
@@ -4322,12 +10291,12 @@ class basic_json
         }
 
         // at only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             return m_value.object->operator[](key);
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
     }
 
     /*!
@@ -4364,13 +10333,13 @@ class basic_json
     const_reference operator[](T* key) const
     {
         // at only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             assert(m_value.object->find(key) != m_value.object->end());
             return m_value.object->find(key)->second;
         }
 
-        JSON_THROW(type_error::create(305, "cannot use operator[] with " + type_name()));
+        JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name())));
     }
 
     /*!
@@ -4423,10 +10392,10 @@ class basic_json
     */
     template<class ValueType, typename std::enable_if<
                  std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0>
-    ValueType value(const typename object_t::key_type& key, ValueType default_value) const
+    ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const
     {
         // at only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             // if key is found, return value and given default value otherwise
             const auto it = find(key);
@@ -4437,10 +10406,8 @@ class basic_json
 
             return default_value;
         }
-        else
-        {
-            JSON_THROW(type_error::create(306, "cannot use value() with " + type_name()));
-        }
+
+        JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name())));
     }
 
     /*!
@@ -4495,10 +10462,10 @@ class basic_json
     */
     template<class ValueType, typename std::enable_if<
                  std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0>
-    ValueType value(const json_pointer& ptr, ValueType default_value) const
+    ValueType value(const json_pointer& ptr, const ValueType& default_value) const
     {
         // at only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             // if pointer resolves a value, return it or use default value
             JSON_TRY
@@ -4511,7 +10478,7 @@ class basic_json
             }
         }
 
-        JSON_THROW(type_error::create(306, "cannot use value() with " + type_name()));
+        JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name())));
     }
 
     /*!
@@ -4662,7 +10629,7 @@ class basic_json
     IteratorType erase(IteratorType pos)
     {
         // make sure iterator fits the current value
-        if (this != pos.m_object)
+        if (JSON_UNLIKELY(this != pos.m_object))
         {
             JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
         }
@@ -4677,7 +10644,7 @@ class basic_json
             case value_t::number_unsigned:
             case value_t::string:
             {
-                if (not pos.m_it.primitive_iterator.is_begin())
+                if (JSON_UNLIKELY(not pos.m_it.primitive_iterator.is_begin()))
                 {
                     JSON_THROW(invalid_iterator::create(205, "iterator out of range"));
                 }
@@ -4708,9 +10675,7 @@ class basic_json
             }
 
             default:
-            {
-                JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name()));
-            }
+                JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
         }
 
         return result;
@@ -4769,7 +10734,7 @@ class basic_json
     IteratorType erase(IteratorType first, IteratorType last)
     {
         // make sure iterator fits the current value
-        if (this != first.m_object or this != last.m_object)
+        if (JSON_UNLIKELY(this != first.m_object or this != last.m_object))
         {
             JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value"));
         }
@@ -4784,7 +10749,8 @@ class basic_json
             case value_t::number_unsigned:
             case value_t::string:
             {
-                if (not first.m_it.primitive_iterator.is_begin() or not last.m_it.primitive_iterator.is_end())
+                if (JSON_LIKELY(not first.m_it.primitive_iterator.is_begin()
+                                or not last.m_it.primitive_iterator.is_end()))
                 {
                     JSON_THROW(invalid_iterator::create(204, "iterators out of range"));
                 }
@@ -4817,9 +10783,7 @@ class basic_json
             }
 
             default:
-            {
-                JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name()));
-            }
+                JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
         }
 
         return result;
@@ -4857,12 +10821,12 @@ class basic_json
     size_type erase(const typename object_t::key_type& key)
     {
         // this erase only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             return m_value.object->erase(key);
         }
 
-        JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name()));
+        JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
     }
 
     /*!
@@ -4892,9 +10856,9 @@ class basic_json
     void erase(const size_type idx)
     {
         // this erase only works for arrays
-        if (is_array())
+        if (JSON_LIKELY(is_array()))
         {
-            if (idx >= size())
+            if (JSON_UNLIKELY(idx >= size()))
             {
                 JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
             }
@@ -4903,7 +10867,7 @@ class basic_json
         }
         else
         {
-            JSON_THROW(type_error::create(307, "cannot use erase() with " + type_name()));
+            JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name())));
         }
     }
 
@@ -5277,10 +11241,6 @@ class basic_json
         return const_reverse_iterator(cbegin());
     }
 
-  private:
-    // forward declaration
-    template<typename IteratorType> class iteration_proxy;
-
   public:
     /*!
     @brief wrapper to access iterator member functions in range-based for
@@ -5319,9 +11279,9 @@ class basic_json
     /// @{
 
     /*!
-    @brief checks whether the container is empty
+    @brief checks whether the container is empty.
 
-    Checks if a JSON value has no elements.
+    Checks if a JSON value has no elements (i.e. whether its @ref size is `0`).
 
     @return The return value depends on the different types and is
             defined as follows:
@@ -5334,23 +11294,27 @@ class basic_json
             object      | result of function `object_t::empty()`
             array       | result of function `array_t::empty()`
 
-    @note This function does not return whether a string stored as JSON value
-    is empty - it returns whether the JSON container itself is empty which is
-    false in the case of a string.
+    @liveexample{The following code uses `empty()` to check if a JSON
+    object contains any elements.,empty}
 
     @complexity Constant, as long as @ref array_t and @ref object_t satisfy
     the Container concept; that is, their `empty()` functions have constant
     complexity.
 
+    @iterators No changes.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @note This function does not return whether a string stored as JSON value
+    is empty - it returns whether the JSON container itself is empty which is
+    false in the case of a string.
+
     @requirement This function helps `basic_json` satisfying the
     [Container](http://en.cppreference.com/w/cpp/concept/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `begin() == end()`.
 
-    @liveexample{The following code uses `empty()` to check if a JSON
-    object contains any elements.,empty}
-
     @sa @ref size() -- returns the number of elements
 
     @since version 1.0.0
@@ -5401,23 +11365,27 @@ class basic_json
             object      | result of function object_t::size()
             array       | result of function array_t::size()
 
-    @note This function does not return the length of a string stored as JSON
-    value - it returns the number of elements in the JSON value which is 1 in
-    the case of a string.
+    @liveexample{The following code calls `size()` on the different value
+    types.,size}
 
     @complexity Constant, as long as @ref array_t and @ref object_t satisfy
     the Container concept; that is, their size() functions have constant
     complexity.
 
+    @iterators No changes.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @note This function does not return the length of a string stored as JSON
+    value - it returns the number of elements in the JSON value which is 1 in
+    the case of a string.
+
     @requirement This function helps `basic_json` satisfying the
     [Container](http://en.cppreference.com/w/cpp/concept/Container)
     requirements:
     - The complexity is constant.
     - Has the semantics of `std::distance(begin(), end())`.
 
-    @liveexample{The following code calls `size()` on the different value
-    types.,size}
-
     @sa @ref empty() -- checks whether the container is empty
     @sa @ref max_size() -- returns the maximal number of elements
 
@@ -5471,10 +11439,17 @@ class basic_json
             object      | result of function `object_t::max_size()`
             array       | result of function `array_t::max_size()`
 
+    @liveexample{The following code calls `max_size()` on the different value
+    types. Note the output is implementation specific.,max_size}
+
     @complexity Constant, as long as @ref array_t and @ref object_t satisfy
     the Container concept; that is, their `max_size()` functions have constant
     complexity.
 
+    @iterators No changes.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
     @requirement This function helps `basic_json` satisfying the
     [Container](http://en.cppreference.com/w/cpp/concept/Container)
     requirements:
@@ -5482,9 +11457,6 @@ class basic_json
     - Has the semantics of returning `b.size()` where `b` is the largest
       possible JSON value.
 
-    @liveexample{The following code calls `max_size()` on the different value
-    types. Note the output is implementation specific.,max_size}
-
     @sa @ref size() -- returns the number of elements
 
     @since version 1.0.0
@@ -5527,7 +11499,8 @@ class basic_json
     @brief clears the contents
 
     Clears the content of a JSON value and resets it to the default value as
-    if @ref basic_json(value_t) would have been called:
+    if @ref basic_json(value_t) would have been called with the current value
+    type from @ref type():
 
     Value type  | initial value
     ----------- | -------------
@@ -5538,11 +11511,24 @@ class basic_json
     object      | `{}`
     array       | `[]`
 
-    @complexity Linear in the size of the JSON value.
+    @post Has the same effect as calling
+    @code {.cpp}
+    *this = basic_json(type());
+    @endcode
 
     @liveexample{The example below shows the effect of `clear()` to different
     JSON types.,clear}
 
+    @complexity Linear in the size of the JSON value.
+
+    @iterators All iterators, pointers and references related to this container
+               are invalidated.
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
+    @sa @ref basic_json(value_t) -- constructor that creates an object with the
+        same value than calling `clear()`
+
     @since version 1.0.0
     */
     void clear() noexcept
@@ -5592,9 +11578,7 @@ class basic_json
             }
 
             default:
-            {
                 break;
-            }
         }
     }
 
@@ -5621,9 +11605,9 @@ class basic_json
     void push_back(basic_json&& val)
     {
         // push_back only works for null objects or arrays
-        if (not(is_null() or is_array()))
+        if (JSON_UNLIKELY(not(is_null() or is_array())))
         {
-            JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name()));
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
         }
 
         // transform null object into an array
@@ -5657,9 +11641,9 @@ class basic_json
     void push_back(const basic_json& val)
     {
         // push_back only works for null objects or arrays
-        if (not(is_null() or is_array()))
+        if (JSON_UNLIKELY(not(is_null() or is_array())))
         {
-            JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name()));
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
         }
 
         // transform null object into an array
@@ -5707,9 +11691,9 @@ class basic_json
     void push_back(const typename object_t::value_type& val)
     {
         // push_back only works for null objects or objects
-        if (not(is_null() or is_object()))
+        if (JSON_UNLIKELY(not(is_null() or is_object())))
         {
-            JSON_THROW(type_error::create(308, "cannot use push_back() with " + type_name()));
+            JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name())));
         }
 
         // transform null object into an object
@@ -5759,12 +11743,13 @@ class basic_json
     @liveexample{The example shows how initializer lists are treated as
     objects when possible.,push_back__initializer_list}
     */
-    void push_back(std::initializer_list<basic_json> init)
+    void push_back(initializer_list_t init)
     {
-        if (is_object() and init.size() == 2 and init.begin()->is_string())
+        if (is_object() and init.size() == 2 and (*init.begin())->is_string())
         {
-            const string_t key = *init.begin();
-            push_back(typename object_t::value_type(key, *(init.begin() + 1)));
+            basic_json&& key = init.begin()->moved_or_copied();
+            push_back(typename object_t::value_type(
+                          std::move(key.get_ref<string_t&>()), (init.begin() + 1)->moved_or_copied()));
         }
         else
         {
@@ -5774,9 +11759,9 @@ class basic_json
 
     /*!
     @brief add an object to an object
-    @copydoc push_back(std::initializer_list<basic_json>)
+    @copydoc push_back(initializer_list_t)
     */
-    reference operator+=(std::initializer_list<basic_json> init)
+    reference operator+=(initializer_list_t init)
     {
         push_back(init);
         return *this;
@@ -5807,9 +11792,9 @@ class basic_json
     void emplace_back(Args&& ... args)
     {
         // emplace_back only works for null objects or arrays
-        if (not(is_null() or is_array()))
+        if (JSON_UNLIKELY(not(is_null() or is_array())))
         {
-            JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + type_name()));
+            JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + std::string(type_name())));
         }
 
         // transform null object into an array
@@ -5855,9 +11840,9 @@ class basic_json
     std::pair<iterator, bool> emplace(Args&& ... args)
     {
         // emplace only works for null objects or arrays
-        if (not(is_null() or is_object()))
+        if (JSON_UNLIKELY(not(is_null() or is_object())))
         {
-            JSON_THROW(type_error::create(311, "cannot use emplace() with " + type_name()));
+            JSON_THROW(type_error::create(311, "cannot use emplace() with " + std::string(type_name())));
         }
 
         // transform null object into an object
@@ -5903,10 +11888,10 @@ class basic_json
     iterator insert(const_iterator pos, const basic_json& val)
     {
         // insert only works for arrays
-        if (is_array())
+        if (JSON_LIKELY(is_array()))
         {
             // check if iterator pos fits to this JSON value
-            if (pos.m_object != this)
+            if (JSON_UNLIKELY(pos.m_object != this))
             {
                 JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
             }
@@ -5917,7 +11902,7 @@ class basic_json
             return result;
         }
 
-        JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
+        JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
     }
 
     /*!
@@ -5956,10 +11941,10 @@ class basic_json
     iterator insert(const_iterator pos, size_type cnt, const basic_json& val)
     {
         // insert only works for arrays
-        if (is_array())
+        if (JSON_LIKELY(is_array()))
         {
             // check if iterator pos fits to this JSON value
-            if (pos.m_object != this)
+            if (JSON_UNLIKELY(pos.m_object != this))
             {
                 JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
             }
@@ -5970,7 +11955,7 @@ class basic_json
             return result;
         }
 
-        JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
+        JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
     }
 
     /*!
@@ -6006,24 +11991,24 @@ class basic_json
     iterator insert(const_iterator pos, const_iterator first, const_iterator last)
     {
         // insert only works for arrays
-        if (not is_array())
+        if (JSON_UNLIKELY(not is_array()))
         {
-            JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
         }
 
         // check if iterator pos fits to this JSON value
-        if (pos.m_object != this)
+        if (JSON_UNLIKELY(pos.m_object != this))
         {
             JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
         }
 
         // check if range iterators belong to the same JSON object
-        if (first.m_object != last.m_object)
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
         {
             JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
         }
 
-        if (first.m_object == this or last.m_object == this)
+        if (JSON_UNLIKELY(first.m_object == this or last.m_object == this))
         {
             JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container"));
         }
@@ -6061,23 +12046,23 @@ class basic_json
 
     @since version 1.0.0
     */
-    iterator insert(const_iterator pos, std::initializer_list<basic_json> ilist)
+    iterator insert(const_iterator pos, initializer_list_t ilist)
     {
         // insert only works for arrays
-        if (not is_array())
+        if (JSON_UNLIKELY(not is_array()))
         {
-            JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
         }
 
         // check if iterator pos fits to this JSON value
-        if (pos.m_object != this)
+        if (JSON_UNLIKELY(pos.m_object != this))
         {
             JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value"));
         }
 
         // insert to array and return iterator
         iterator result(this);
-        result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist);
+        result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist.begin(), ilist.end());
         return result;
     }
 
@@ -6107,19 +12092,20 @@ class basic_json
     void insert(const_iterator first, const_iterator last)
     {
         // insert only works for objects
-        if (not is_object())
+        if (JSON_UNLIKELY(not is_object()))
         {
-            JSON_THROW(type_error::create(309, "cannot use insert() with " + type_name()));
+            JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name())));
         }
 
         // check if range iterators belong to the same JSON object
-        if (first.m_object != last.m_object)
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
         {
             JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
         }
 
         // passed iterators must belong to objects
-        if (not first.m_object->is_object() or not first.m_object->is_object())
+        if (JSON_UNLIKELY(not first.m_object->is_object()
+                          or not last.m_object->is_object()))
         {
             JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
         }
@@ -6127,6 +12113,110 @@ class basic_json
         m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator);
     }
 
+    /*!
+    @brief updates a JSON object from another object, overwriting existing keys
+
+    Inserts all values from JSON object @a j and overwrites existing keys.
+
+    @param[in] j  JSON object to read values from
+
+    @throw type_error.312 if called on JSON values other than objects; example:
+    `"cannot use update() with string"`
+
+    @complexity O(N*log(size() + N)), where N is the number of elements to
+                insert.
+
+    @liveexample{The example shows how `update()` is used.,update}
+
+    @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
+
+    @since version 3.0.0
+    */
+    void update(const_reference j)
+    {
+        // implicitly convert null value to an empty object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value.object = create<object_t>();
+            assert_invariant();
+        }
+
+        if (JSON_UNLIKELY(not is_object()))
+        {
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name())));
+        }
+        if (JSON_UNLIKELY(not j.is_object()))
+        {
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(j.type_name())));
+        }
+
+        for (auto it = j.begin(); it != j.end(); ++it)
+        {
+            m_value.object->operator[](it.key()) = it.value();
+        }
+    }
+
+    /*!
+    @brief updates a JSON object from another object, overwriting existing keys
+
+    Inserts all values from from range `[first, last)` and overwrites existing
+    keys.
+
+    @param[in] first begin of the range of elements to insert
+    @param[in] last end of the range of elements to insert
+
+    @throw type_error.312 if called on JSON values other than objects; example:
+    `"cannot use update() with string"`
+    @throw invalid_iterator.202 if iterator @a first or @a last does does not
+    point to an object; example: `"iterators first and last must point to
+    objects"`
+    @throw invalid_iterator.210 if @a first and @a last do not belong to the
+    same JSON value; example: `"iterators do not fit"`
+
+    @complexity O(N*log(size() + N)), where N is the number of elements to
+                insert.
+
+    @liveexample{The example shows how `update()` is used__range.,update}
+
+    @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
+
+    @since version 3.0.0
+    */
+    void update(const_iterator first, const_iterator last)
+    {
+        // implicitly convert null value to an empty object
+        if (is_null())
+        {
+            m_type = value_t::object;
+            m_value.object = create<object_t>();
+            assert_invariant();
+        }
+
+        if (JSON_UNLIKELY(not is_object()))
+        {
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name())));
+        }
+
+        // check if range iterators belong to the same JSON object
+        if (JSON_UNLIKELY(first.m_object != last.m_object))
+        {
+            JSON_THROW(invalid_iterator::create(210, "iterators do not fit"));
+        }
+
+        // passed iterators must belong to objects
+        if (JSON_UNLIKELY(not first.m_object->is_object()
+                          or not first.m_object->is_object()))
+        {
+            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects"));
+        }
+
+        for (auto it = first; it != last; ++it)
+        {
+            m_value.object->operator[](it.key()) = it.value();
+        }
+    }
+
     /*!
     @brief exchanges the values
 
@@ -6179,13 +12269,13 @@ class basic_json
     void swap(array_t& other)
     {
         // swap only works for arrays
-        if (is_array())
+        if (JSON_LIKELY(is_array()))
         {
             std::swap(*(m_value.array), other);
         }
         else
         {
-            JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name()));
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
         }
     }
 
@@ -6212,13 +12302,13 @@ class basic_json
     void swap(object_t& other)
     {
         // swap only works for objects
-        if (is_object())
+        if (JSON_LIKELY(is_object()))
         {
             std::swap(*(m_value.object), other);
         }
         else
         {
-            JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name()));
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
         }
     }
 
@@ -6245,13 +12335,13 @@ class basic_json
     void swap(string_t& other)
     {
         // swap only works for strings
-        if (is_string())
+        if (JSON_LIKELY(is_string()))
         {
             std::swap(*(m_value.string), other);
         }
         else
         {
-            JSON_THROW(type_error::create(310, "cannot use swap() with " + type_name()));
+            JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name())));
         }
     }
 
@@ -6273,18 +12363,30 @@ class basic_json
       their stored values are the same according to their respective
       `operator==`.
     - Integer and floating-point numbers are automatically converted before
-      comparison. Floating-point numbers are compared indirectly: two
-      floating-point numbers `f1` and `f2` are considered equal if neither
-      `f1 > f2` nor `f2 > f1` holds. Note than two NaN values are always
-      treated as unequal.
+      comparison. Note than two NaN values are always treated as unequal.
     - Two JSON null values are equal.
 
+    @note Floating-point inside JSON values numbers are compared with
+    `json::number_float_t::operator==` which is `double::operator==` by
+    default. To compare floating-point while respecting an epsilon, an alternative
+    [comparison function](https://github.com/mariokonrad/marnav/blob/master/src/marnav/math/floatingpoint.hpp#L34-#L39)
+    could be used, for instance
+    @code {.cpp}
+    template <typename T, typename = typename std::enable_if<std::is_floating_point<T>::value, T>::type>
+    inline bool is_same(T a, T b, T epsilon = std::numeric_limits<T>::epsilon()) noexcept
+    {
+        return std::abs(a - b) <= epsilon;
+    }
+    @endcode
+
     @note NaN values never compare equal to themselves or to other NaN values.
 
     @param[in] lhs  first JSON value to consider
     @param[in] rhs  second JSON value to consider
     @return whether the values @a lhs and @a rhs are equal
 
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
     @complexity Linear.
 
     @liveexample{The example demonstrates comparing several JSON
@@ -6302,66 +12404,56 @@ class basic_json
             switch (lhs_type)
             {
                 case value_t::array:
-                {
-                    return *lhs.m_value.array == *rhs.m_value.array;
-                }
+                    return (*lhs.m_value.array == *rhs.m_value.array);
+
                 case value_t::object:
-                {
-                    return *lhs.m_value.object == *rhs.m_value.object;
-                }
+                    return (*lhs.m_value.object == *rhs.m_value.object);
+
                 case value_t::null:
-                {
                     return true;
-                }
+
                 case value_t::string:
-                {
-                    return *lhs.m_value.string == *rhs.m_value.string;
-                }
+                    return (*lhs.m_value.string == *rhs.m_value.string);
+
                 case value_t::boolean:
-                {
-                    return lhs.m_value.boolean == rhs.m_value.boolean;
-                }
+                    return (lhs.m_value.boolean == rhs.m_value.boolean);
+
                 case value_t::number_integer:
-                {
-                    return lhs.m_value.number_integer == rhs.m_value.number_integer;
-                }
+                    return (lhs.m_value.number_integer == rhs.m_value.number_integer);
+
                 case value_t::number_unsigned:
-                {
-                    return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned;
-                }
+                    return (lhs.m_value.number_unsigned == rhs.m_value.number_unsigned);
+
                 case value_t::number_float:
-                {
-                    return lhs.m_value.number_float == rhs.m_value.number_float;
-                }
+                    return (lhs.m_value.number_float == rhs.m_value.number_float);
+
                 default:
-                {
                     return false;
-                }
             }
         }
         else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float)
         {
-            return static_cast<number_float_t>(lhs.m_value.number_integer) == rhs.m_value.number_float;
+            return (static_cast<number_float_t>(lhs.m_value.number_integer) == rhs.m_value.number_float);
         }
         else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer)
         {
-            return lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_integer);
+            return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_integer));
         }
         else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float)
         {
-            return static_cast<number_float_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_float;
+            return (static_cast<number_float_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_float);
         }
         else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned)
         {
-            return lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_unsigned);
+            return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_unsigned));
         }
         else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer)
         {
-            return static_cast<number_integer_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_integer;
+            return (static_cast<number_integer_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_integer);
         }
         else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned)
         {
-            return lhs.m_value.number_integer == static_cast<number_integer_t>(rhs.m_value.number_unsigned);
+            return (lhs.m_value.number_integer == static_cast<number_integer_t>(rhs.m_value.number_unsigned));
         }
 
         return false;
@@ -6400,6 +12492,8 @@ class basic_json
 
     @complexity Linear.
 
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
     @liveexample{The example demonstrates comparing several JSON
     types.,operator__notequal}
 
@@ -6451,6 +12545,8 @@ class basic_json
 
     @complexity Linear.
 
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
     @liveexample{The example demonstrates comparing several JSON
     types.,operator__less}
 
@@ -6466,41 +12562,31 @@ class basic_json
             switch (lhs_type)
             {
                 case value_t::array:
-                {
                     return (*lhs.m_value.array) < (*rhs.m_value.array);
-                }
+
                 case value_t::object:
-                {
                     return *lhs.m_value.object < *rhs.m_value.object;
-                }
+
                 case value_t::null:
-                {
                     return false;
-                }
+
                 case value_t::string:
-                {
                     return *lhs.m_value.string < *rhs.m_value.string;
-                }
+
                 case value_t::boolean:
-                {
                     return lhs.m_value.boolean < rhs.m_value.boolean;
-                }
+
                 case value_t::number_integer:
-                {
                     return lhs.m_value.number_integer < rhs.m_value.number_integer;
-                }
+
                 case value_t::number_unsigned:
-                {
                     return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned;
-                }
+
                 case value_t::number_float:
-                {
                     return lhs.m_value.number_float < rhs.m_value.number_float;
-                }
+
                 default:
-                {
                     return false;
-                }
             }
         }
         else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float)
@@ -6568,6 +12654,8 @@ class basic_json
 
     @complexity Linear.
 
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
     @liveexample{The example demonstrates comparing several JSON
     types.,operator__greater}
 
@@ -6612,6 +12700,8 @@ class basic_json
 
     @complexity Linear.
 
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
     @liveexample{The example demonstrates comparing several JSON
     types.,operator__lessequal}
 
@@ -6656,6 +12746,8 @@ class basic_json
 
     @complexity Linear.
 
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
+
     @liveexample{The example demonstrates comparing several JSON
     types.,operator__greaterequal}
 
@@ -6690,110 +12782,6 @@ class basic_json
 
     /// @}
 
-  private:
-    /////////////////////
-    // output adapters //
-    /////////////////////
-
-    /// abstract output adapter interface
-    template<typename CharType>
-    class output_adapter
-    {
-      public:
-        virtual void write_character(CharType c) = 0;
-        virtual void write_characters(const CharType* s, size_t length) = 0;
-        virtual ~output_adapter() {}
-
-        static std::shared_ptr<output_adapter<CharType>> create(std::vector<CharType>& vec)
-        {
-            return std::shared_ptr<output_adapter>(new output_vector_adapter<CharType>(vec));
-        }
-
-        static std::shared_ptr<output_adapter<CharType>> create(std::ostream& s)
-        {
-            return std::shared_ptr<output_adapter>(new output_stream_adapter<CharType>(s));
-        }
-
-        static std::shared_ptr<output_adapter<CharType>> create(std::string& s)
-        {
-            return std::shared_ptr<output_adapter>(new output_string_adapter<CharType>(s));
-        }
-    };
-
-    /// a type to simplify interfaces
-    template<typename CharType>
-    using output_adapter_t = std::shared_ptr<output_adapter<CharType>>;
-
-    /// output adapter for byte vectors
-    template<typename CharType>
-    class output_vector_adapter : public output_adapter<CharType>
-    {
-      public:
-        output_vector_adapter(std::vector<CharType>& vec)
-            : v(vec)
-        {}
-
-        void write_character(CharType c) override
-        {
-            v.push_back(c);
-        }
-
-        void write_characters(const CharType* s, size_t length) override
-        {
-            std::copy(s, s + length, std::back_inserter(v));
-        }
-
-      private:
-        std::vector<CharType>& v;
-    };
-
-    /// putput adatpter for output streams
-    template<typename CharType>
-    class output_stream_adapter : public output_adapter<CharType>
-    {
-      public:
-        output_stream_adapter(std::basic_ostream<CharType>& s)
-            : stream(s)
-        {}
-
-        void write_character(CharType c) override
-        {
-            stream.put(c);
-        }
-
-        void write_characters(const CharType* s, size_t length) override
-        {
-            stream.write(s, static_cast<std::streamsize>(length));
-        }
-
-      private:
-        std::basic_ostream<CharType>& stream;
-    };
-
-    /// output adapter for basic_string
-    template<typename CharType>
-    class output_string_adapter : public output_adapter<CharType>
-    {
-      public:
-        output_string_adapter(std::string& s)
-            : str(s)
-        {}
-
-        void write_character(CharType c) override
-        {
-            str.push_back(c);
-        }
-
-        void write_characters(const CharType* s, size_t length) override
-        {
-            str.append(s, length);
-        }
-
-      private:
-        std::basic_string<CharType>& str;
-    };
-
-
     ///////////////////
     // serialization //
     ///////////////////
@@ -6801,598 +12789,6 @@ class basic_json
     /// @name serialization
     /// @{
 
-  private:
-    /*!
-    @brief wrapper around the serialization functions
-    */
-    class serializer
-    {
-      public:
-        /*!
-        @param[in] s  output stream to serialize to
-        @param[in] ichar  indentation character to use
-        */
-        serializer(output_adapter_t<char> s, const char ichar)
-            : o(s), loc(std::localeconv()),
-              thousands_sep(!loc->thousands_sep ? '\0' : loc->thousands_sep[0]),
-              decimal_point(!loc->decimal_point ? '\0' : loc->decimal_point[0]),
-              indent_char(ichar), indent_string(512, indent_char)
-        {}
-
-        // delete because of pointer members
-        serializer(const serializer&) = delete;
-        serializer& operator=(const serializer&) = delete;
-
-        /*!
-        @brief internal implementation of the serialization function
-
-        This function is called by the public member function dump and
-        organizes the serialization internally. The indentation level is
-        propagated as additional parameter. In case of arrays and objects, the
-        function is called recursively.
-
-        - strings and object keys are escaped using `escape_string()`
-        - integer numbers are converted implicitly via `operator<<`
-        - floating-point numbers are converted to a string using `"%g"` format
-
-        @param[in] val             value to serialize
-        @param[in] pretty_print    whether the output shall be pretty-printed
-        @param[in] indent_step     the indent level
-        @param[in] current_indent  the current indent level (only used internally)
-        */
-        void dump(const basic_json& val,
-                  const bool pretty_print,
-                  const unsigned int indent_step,
-                  const unsigned int current_indent = 0)
-        {
-            switch (val.m_type)
-            {
-                case value_t::object:
-                {
-                    if (val.m_value.object->empty())
-                    {
-                        o->write_characters("{}", 2);
-                        return;
-                    }
-
-                    if (pretty_print)
-                    {
-                        o->write_characters("{\n", 2);
-
-                        // variable to hold indentation for recursive calls
-                        const auto new_indent = current_indent + indent_step;
-                        if (indent_string.size() < new_indent)
-                        {
-                            indent_string.resize(new_indent, ' ');
-                        }
-
-                        // first n-1 elements
-                        auto i = val.m_value.object->cbegin();
-                        for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
-                        {
-                            o->write_characters(indent_string.c_str(), new_indent);
-                            o->write_character('\"');
-                            dump_escaped(i->first);
-                            o->write_characters("\": ", 3);
-                            dump(i->second, true, indent_step, new_indent);
-                            o->write_characters(",\n", 2);
-                        }
-
-                        // last element
-                        assert(i != val.m_value.object->cend());
-                        o->write_characters(indent_string.c_str(), new_indent);
-                        o->write_character('\"');
-                        dump_escaped(i->first);
-                        o->write_characters("\": ", 3);
-                        dump(i->second, true, indent_step, new_indent);
-
-                        o->write_character('\n');
-                        o->write_characters(indent_string.c_str(), current_indent);
-                        o->write_character('}');
-                    }
-                    else
-                    {
-                        o->write_character('{');
-
-                        // first n-1 elements
-                        auto i = val.m_value.object->cbegin();
-                        for (size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i)
-                        {
-                            o->write_character('\"');
-                            dump_escaped(i->first);
-                            o->write_characters("\":", 2);
-                            dump(i->second, false, indent_step, current_indent);
-                            o->write_character(',');
-                        }
-
-                        // last element
-                        assert(i != val.m_value.object->cend());
-                        o->write_character('\"');
-                        dump_escaped(i->first);
-                        o->write_characters("\":", 2);
-                        dump(i->second, false, indent_step, current_indent);
-
-                        o->write_character('}');
-                    }
-
-                    return;
-                }
-
-                case value_t::array:
-                {
-                    if (val.m_value.array->empty())
-                    {
-                        o->write_characters("[]", 2);
-                        return;
-                    }
-
-                    if (pretty_print)
-                    {
-                        o->write_characters("[\n", 2);
-
-                        // variable to hold indentation for recursive calls
-                        const auto new_indent = current_indent + indent_step;
-                        if (indent_string.size() < new_indent)
-                        {
-                            indent_string.resize(new_indent, ' ');
-                        }
-
-                        // first n-1 elements
-                        for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i)
-                        {
-                            o->write_characters(indent_string.c_str(), new_indent);
-                            dump(*i, true, indent_step, new_indent);
-                            o->write_characters(",\n", 2);
-                        }
-
-                        // last element
-                        assert(not val.m_value.array->empty());
-                        o->write_characters(indent_string.c_str(), new_indent);
-                        dump(val.m_value.array->back(), true, indent_step, new_indent);
-
-                        o->write_character('\n');
-                        o->write_characters(indent_string.c_str(), current_indent);
-                        o->write_character(']');
-                    }
-                    else
-                    {
-                        o->write_character('[');
-
-                        // first n-1 elements
-                        for (auto i = val.m_value.array->cbegin(); i != val.m_value.array->cend() - 1; ++i)
-                        {
-                            dump(*i, false, indent_step, current_indent);
-                            o->write_character(',');
-                        }
-
-                        // last element
-                        assert(not val.m_value.array->empty());
-                        dump(val.m_value.array->back(), false, indent_step, current_indent);
-
-                        o->write_character(']');
-                    }
-
-                    return;
-                }
-
-                case value_t::string:
-                {
-                    o->write_character('\"');
-                    dump_escaped(*val.m_value.string);
-                    o->write_character('\"');
-                    return;
-                }
-
-                case value_t::boolean:
-                {
-                    if (val.m_value.boolean)
-                    {
-                        o->write_characters("true", 4);
-                    }
-                    else
-                    {
-                        o->write_characters("false", 5);
-                    }
-                    return;
-                }
-
-                case value_t::number_integer:
-                {
-                    dump_integer(val.m_value.number_integer);
-                    return;
-                }
-
-                case value_t::number_unsigned:
-                {
-                    dump_integer(val.m_value.number_unsigned);
-                    return;
-                }
-
-                case value_t::number_float:
-                {
-                    dump_float(val.m_value.number_float);
-                    return;
-                }
-
-                case value_t::discarded:
-                {
-                    o->write_characters("<discarded>", 11);
-                    return;
-                }
-
-                case value_t::null:
-                {
-                    o->write_characters("null", 4);
-                    return;
-                }
-            }
-        }
-
-      private:
-        /*!
-        @brief calculates the extra space to escape a JSON string
-
-        @param[in] s  the string to escape
-        @return the number of characters required to escape string @a s
-
-        @complexity Linear in the length of string @a s.
-        */
-        static std::size_t extra_space(const string_t& s) noexcept
-        {
-            return std::accumulate(s.begin(), s.end(), size_t{},
-                                   [](size_t res, typename string_t::value_type c)
-            {
-                switch (c)
-                {
-                    case '"':
-                    case '\\':
-                    case '\b':
-                    case '\f':
-                    case '\n':
-                    case '\r':
-                    case '\t':
-                    {
-                        // from c (1 byte) to \x (2 bytes)
-                        return res + 1;
-                    }
-
-                    case 0x00:
-                    case 0x01:
-                    case 0x02:
-                    case 0x03:
-                    case 0x04:
-                    case 0x05:
-                    case 0x06:
-                    case 0x07:
-                    case 0x0b:
-                    case 0x0e:
-                    case 0x0f:
-                    case 0x10:
-                    case 0x11:
-                    case 0x12:
-                    case 0x13:
-                    case 0x14:
-                    case 0x15:
-                    case 0x16:
-                    case 0x17:
-                    case 0x18:
-                    case 0x19:
-                    case 0x1a:
-                    case 0x1b:
-                    case 0x1c:
-                    case 0x1d:
-                    case 0x1e:
-                    case 0x1f:
-                    {
-                        // from c (1 byte) to \uxxxx (6 bytes)
-                        return res + 5;
-                    }
-
-                    default:
-                    {
-                        return res;
-                    }
-                }
-            });
-        }
-
-        /*!
-        @brief dump escaped string
-
-        Escape a string by replacing certain special characters by a sequence
-        of an escape character (backslash) and another character and other
-        control characters by a sequence of "\u" followed by a four-digit hex
-        representation. The escaped string is written to output stream @a o.
-
-        @param[in] s  the string to escape
-
-        @complexity Linear in the length of string @a s.
-        */
-        void dump_escaped(const string_t& s) const
-        {
-            const auto space = extra_space(s);
-            if (space == 0)
-            {
-                o->write_characters(s.c_str(), s.size());
-                return;
-            }
-
-            // create a result string of necessary size
-            string_t result(s.size() + space, '\\');
-            std::size_t pos = 0;
-
-            for (const auto& c : s)
-            {
-                switch (c)
-                {
-                    // quotation mark (0x22)
-                    case '"':
-                    {
-                        result[pos + 1] = '"';
-                        pos += 2;
-                        break;
-                    }
-
-                    // reverse solidus (0x5c)
-                    case '\\':
-                    {
-                        // nothing to change
-                        pos += 2;
-                        break;
-                    }
-
-                    // backspace (0x08)
-                    case '\b':
-                    {
-                        result[pos + 1] = 'b';
-                        pos += 2;
-                        break;
-                    }
-
-                    // formfeed (0x0c)
-                    case '\f':
-                    {
-                        result[pos + 1] = 'f';
-                        pos += 2;
-                        break;
-                    }
-
-                    // newline (0x0a)
-                    case '\n':
-                    {
-                        result[pos + 1] = 'n';
-                        pos += 2;
-                        break;
-                    }
-
-                    // carriage return (0x0d)
-                    case '\r':
-                    {
-                        result[pos + 1] = 'r';
-                        pos += 2;
-                        break;
-                    }
-
-                    // horizontal tab (0x09)
-                    case '\t':
-                    {
-                        result[pos + 1] = 't';
-                        pos += 2;
-                        break;
-                    }
-
-                    case 0x00:
-                    case 0x01:
-                    case 0x02:
-                    case 0x03:
-                    case 0x04:
-                    case 0x05:
-                    case 0x06:
-                    case 0x07:
-                    case 0x0b:
-                    case 0x0e:
-                    case 0x0f:
-                    case 0x10:
-                    case 0x11:
-                    case 0x12:
-                    case 0x13:
-                    case 0x14:
-                    case 0x15:
-                    case 0x16:
-                    case 0x17:
-                    case 0x18:
-                    case 0x19:
-                    case 0x1a:
-                    case 0x1b:
-                    case 0x1c:
-                    case 0x1d:
-                    case 0x1e:
-                    case 0x1f:
-                    {
-                        // convert a number 0..15 to its hex representation
-                        // (0..f)
-                        static const char hexify[16] =
-                        {
-                            '0', '1', '2', '3', '4', '5', '6', '7',
-                            '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
-                        };
-
-                        // print character c as \uxxxx
-                        for (const char m :
-                    { 'u', '0', '0', hexify[c >> 4], hexify[c & 0x0f]
-                        })
-                        {
-                            result[++pos] = m;
-                        }
-
-                        ++pos;
-                        break;
-                    }
-
-                    default:
-                    {
-                        // all other characters are added as-is
-                        result[pos++] = c;
-                        break;
-                    }
-                }
-            }
-
-            assert(pos == s.size() + space);
-            o->write_characters(result.c_str(), result.size());
-        }
-
-        /*!
-        @brief dump an integer
-
-        Dump a given integer to output stream @a o. Works internally with
-        @a number_buffer.
-
-        @param[in] x  integer number (signed or unsigned) to dump
-        @tparam NumberType either @a number_integer_t or @a number_unsigned_t
-        */
-        template<typename NumberType, detail::enable_if_t <
-                     std::is_same<NumberType, number_unsigned_t>::value or
-                     std::is_same<NumberType, number_integer_t>::value, int> = 0>
-        void dump_integer(NumberType x)
-        {
-            // special case for "0"
-            if (x == 0)
-            {
-                o->write_character('0');
-                return;
-            }
-
-            const bool is_negative = x < 0;
-            size_t i = 0;
-
-            // spare 1 byte for '\0'
-            while (x != 0 and i < number_buffer.size() - 1)
-            {
-                const auto digit = std::labs(static_cast<long>(x % 10));
-                number_buffer[i++] = static_cast<char>('0' + digit);
-                x /= 10;
-            }
-
-            // make sure the number has been processed completely
-            assert(x == 0);
-
-            if (is_negative)
-            {
-                // make sure there is capacity for the '-'
-                assert(i < number_buffer.size() - 2);
-                number_buffer[i++] = '-';
-            }
-
-            std::reverse(number_buffer.begin(), number_buffer.begin() + i);
-            o->write_characters(number_buffer.data(), i);
-        }
-
-        /*!
-        @brief dump a floating-point number
-
-        Dump a given floating-point number to output stream @a o. Works
-        internally with @a number_buffer.
-
-        @param[in] x  floating-point number to dump
-        */
-        void dump_float(number_float_t x)
-        {
-            // NaN / inf
-            if (not std::isfinite(x) or std::isnan(x))
-            {
-                o->write_characters("null", 4);
-                return;
-            }
-
-            // special case for 0.0 and -0.0
-            if (x == 0)
-            {
-                if (std::signbit(x))
-                {
-                    o->write_characters("-0.0", 4);
-                }
-                else
-                {
-                    o->write_characters("0.0", 3);
-                }
-                return;
-            }
-
-            // get number of digits for a text -> float -> text round-trip
-            static constexpr auto d = std::numeric_limits<number_float_t>::digits10;
-
-            // the actual conversion
-            std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(),
-                                          "%.*g", d, x);
-
-            // negative value indicates an error
-            assert(len > 0);
-            // check if buffer was large enough
-            assert(static_cast<size_t>(len) < number_buffer.size());
-
-            // erase thousands separator
-            if (thousands_sep != '\0')
-            {
-                const auto end = std::remove(number_buffer.begin(),
-                                             number_buffer.begin() + len,
-                                             thousands_sep);
-                std::fill(end, number_buffer.end(), '\0');
-                assert((end - number_buffer.begin()) <= len);
-                len = (end - number_buffer.begin());
-            }
-
-            // convert decimal point to '.'
-            if (decimal_point != '\0' and decimal_point != '.')
-            {
-                for (auto& c : number_buffer)
-                {
-                    if (c == decimal_point)
-                    {
-                        c = '.';
-                        break;
-                    }
-                }
-            }
-
-            o->write_characters(number_buffer.data(), static_cast<size_t>(len));
-
-            // determine if need to append ".0"
-            const bool value_is_int_like = std::none_of(number_buffer.begin(),
-                                           number_buffer.begin() + len + 1,
-                                           [](char c)
-            {
-                return c == '.' or c == 'e';
-            });
-
-            if (value_is_int_like)
-            {
-                o->write_characters(".0", 2);
-            }
-        }
-
-      private:
-        /// the output of the serializer
-        output_adapter_t<char> o = nullptr;
-
-        /// a (hopefully) large enough character buffer
-        std::array<char, 64> number_buffer{{}};
-
-        /// the locale
-        const std::lconv* loc = nullptr;
-        /// the locale's thousand separator character
-        const char thousands_sep = '\0';
-        /// the locale's decimal point character
-        const char decimal_point = '\0';
-
-        /// the indentation character
-        const char indent_char;
-
-        /// the indentation string
-        string_t indent_string;
-    };
-
-  public:
     /*!
     @brief serialize to stream
 
@@ -7431,8 +12827,8 @@ class basic_json
         o.width(0);
 
         // do the actual serialization
-        serializer s(output_adapter<char>::create(o), o.fill());
-        s.dump(j, pretty_print, static_cast<unsigned int>(indentation));
+        serializer s(detail::output_adapter<char>(o), o.fill());
+        s.dump(j, pretty_print, false, static_cast<unsigned int>(indentation));
         return o;
     }
 
@@ -7440,7 +12836,7 @@ class basic_json
     @brief serialize to stream
     @deprecated This stream operator is deprecated and will be removed in a
                 future version of the library. Please use
-                @ref std::ostream& operator<<(std::ostream&, const basic_json&)
+                @ref operator<<(std::ostream&, const basic_json&)
                 instead; that is, replace calls like `j >> o;` with `o << j;`.
     */
     JSON_DEPRECATED
@@ -7460,15 +12856,36 @@ class basic_json
     /// @{
 
     /*!
-    @brief deserialize from an array
+    @brief deserialize from a compatible input
 
-    This function reads from an array of 1-byte values.
+    This function reads from a compatible input. Examples are:
+    - an array of 1-byte values
+    - strings with character/literal type with size of 1 byte
+    - input streams
+    - container with contiguous storage of 1-byte values. Compatible container
+      types include `std::vector`, `std::string`, `std::array`,
+      `std::valarray`, and `std::initializer_list`. Furthermore, C-style
+      arrays can be used with `std::begin()`/`std::end()`. User-defined
+      containers can be used as long as they implement random-access iterators
+      and a contiguous storage.
 
     @pre Each element of the container has a size of 1 byte. Violating this
     precondition yields undefined behavior. **This precondition is enforced
     with a static assertion.**
 
-    @param[in] array  array to read from
+    @pre The container storage is contiguous. Violating this precondition
+    yields undefined behavior. **This precondition is enforced with an
+    assertion.**
+    @pre Each element of the container has a size of 1 byte. Violating this
+    precondition yields undefined behavior. **This precondition is enforced
+    with a static assertion.**
+
+    @warning There is no way to enforce all preconditions at compile-time. If
+             the function is called with a noncompliant container and with
+             assertions switched off, the behavior is undefined and will most
+             likely yield segmentation violation.
+
+    @param[in] i  input to read from
     @param[in] cb  a parser callback function of type @ref parser_callback_t
     which is used to control the deserialization by filtering unwanted values
     (optional)
@@ -7489,125 +12906,46 @@ class basic_json
     @liveexample{The example below demonstrates the `parse()` function reading
     from an array.,parse__array__parser_callback_t}
 
-    @since version 2.0.3
-    */
-    template<class T, std::size_t N>
-    static basic_json parse(T (&array)[N],
-                            const parser_callback_t cb = nullptr)
-    {
-        // delegate the call to the iterator-range parse overload
-        return parse(std::begin(array), std::end(array), cb);
-    }
-
-    template<class T, std::size_t N>
-    static bool accept(T (&array)[N])
-    {
-        // delegate the call to the iterator-range accept overload
-        return accept(std::begin(array), std::end(array));
-    }
-
-    /*!
-    @brief deserialize from string literal
-
-    @tparam CharT character/literal type with size of 1 byte
-    @param[in] s  string literal to read a serialized JSON value from
-    @param[in] cb a parser callback function of type @ref parser_callback_t
-    which is used to control the deserialization by filtering unwanted values
-    (optional)
-
-    @return result of the deserialization
-
-    @throw parse_error.101 in case of an unexpected token
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-
-    @complexity Linear in the length of the input. The parser is a predictive
-    LL(1) parser. The complexity can be higher if the parser callback function
-    @a cb has a super-linear complexity.
-
-    @note A UTF-8 byte order mark is silently ignored.
-    @note String containers like `std::string` or @ref string_t can be parsed
-          with @ref parse(const ContiguousContainer&, const parser_callback_t)
-
     @liveexample{The example below demonstrates the `parse()` function with
     and without callback function.,parse__string__parser_callback_t}
 
-    @sa @ref parse(std::istream&, const parser_callback_t) for a version that
-    reads from an input stream
-
-    @since version 1.0.0 (originally for @ref string_t)
-    */
-    template<typename CharT, typename std::enable_if<
-                 std::is_pointer<CharT>::value and
-                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
-                 sizeof(typename std::remove_pointer<CharT>::type) == 1, int>::type = 0>
-    static basic_json parse(const CharT s,
-                            const parser_callback_t cb = nullptr)
-    {
-        return parser(input_adapter::create(s), cb).parse(true);
-    }
-
-    template<typename CharT, typename std::enable_if<
-                 std::is_pointer<CharT>::value and
-                 std::is_integral<typename std::remove_pointer<CharT>::type>::value and
-                 sizeof(typename std::remove_pointer<CharT>::type) == 1, int>::type = 0>
-    static bool accept(const CharT s)
-    {
-        return parser(input_adapter::create(s)).accept(true);
-    }
-
-    /*!
-    @brief deserialize from stream
-
-    @param[in,out] i  stream to read a serialized JSON value from
-    @param[in] cb a parser callback function of type @ref parser_callback_t
-    which is used to control the deserialization by filtering unwanted values
-    (optional)
-
-    @return result of the deserialization
-
-    @throw parse_error.101 in case of an unexpected token
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-    @throw parse_error.111 if input stream is in a bad state
-
-    @complexity Linear in the length of the input. The parser is a predictive
-    LL(1) parser. The complexity can be higher if the parser callback function
-    @a cb has a super-linear complexity.
-
-    @note A UTF-8 byte order mark is silently ignored.
-
     @liveexample{The example below demonstrates the `parse()` function with
     and without callback function.,parse__istream__parser_callback_t}
 
-    @sa @ref parse(const CharT, const parser_callback_t) for a version
-    that reads from a string
+    @liveexample{The example below demonstrates the `parse()` function reading
+    from a contiguous container.,parse__contiguouscontainer__parser_callback_t}
 
-    @since version 1.0.0
+    @since version 2.0.3 (contiguous containers)
     */
-    static basic_json parse(std::istream& i,
-                            const parser_callback_t cb = nullptr)
+    static basic_json parse(detail::input_adapter i,
+                            const parser_callback_t cb = nullptr,
+                            const bool allow_exceptions = true)
     {
-        return parser(input_adapter::create(i), cb).parse(true);
-    }
-
-    static bool accept(std::istream& i)
-    {
-        return parser(input_adapter::create(i)).accept(true);
+        basic_json result;
+        parser(i, cb, allow_exceptions).parse(true, result);
+        return result;
     }
 
     /*!
-    @copydoc parse(std::istream&, const parser_callback_t)
+    @copydoc basic_json parse(detail::input_adapter, const parser_callback_t)
     */
-    static basic_json parse(std::istream&& i,
-                            const parser_callback_t cb = nullptr)
+    static basic_json parse(detail::input_adapter& i,
+                            const parser_callback_t cb = nullptr,
+                            const bool allow_exceptions = true)
     {
-        return parser(input_adapter::create(i), cb).parse(true);
+        basic_json result;
+        parser(i, cb, allow_exceptions).parse(true, result);
+        return result;
     }
 
-    static bool accept(std::istream&& i)
+    static bool accept(detail::input_adapter i)
     {
-        return parser(input_adapter::create(i)).accept(true);
+        return parser(i).accept(true);
+    }
+
+    static bool accept(detail::input_adapter& i)
+    {
+        return parser(i).accept(true);
     }
 
     /*!
@@ -7660,9 +12998,12 @@ class basic_json
                      std::random_access_iterator_tag,
                      typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
     static basic_json parse(IteratorType first, IteratorType last,
-                            const parser_callback_t cb = nullptr)
+                            const parser_callback_t cb = nullptr,
+                            const bool allow_exceptions = true)
     {
-        return parser(input_adapter::create(first, last), cb).parse(true);
+        basic_json result;
+        parser(detail::input_adapter(first, last), cb, allow_exceptions).parse(true, result);
+        return result;
     }
 
     template<class IteratorType, typename std::enable_if<
@@ -7671,90 +13012,20 @@ class basic_json
                      typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0>
     static bool accept(IteratorType first, IteratorType last)
     {
-        return parser(input_adapter::create(first, last)).accept(true);
-    }
-
-    /*!
-    @brief deserialize from a container with contiguous storage
-
-    This function reads from a container with contiguous storage of 1-byte
-    values. Compatible container types include `std::vector`, `std::string`,
-    `std::array`, and `std::initializer_list`. User-defined containers can be
-    used as long as they implement random-access iterators and a contiguous
-    storage.
-
-    @pre The container storage is contiguous. Violating this precondition
-    yields undefined behavior. **This precondition is enforced with an
-    assertion.**
-    @pre Each element of the container has a size of 1 byte. Violating this
-    precondition yields undefined behavior. **This precondition is enforced
-    with a static assertion.**
-
-    @warning There is no way to enforce all preconditions at compile-time. If
-             the function is called with a noncompliant container and with
-             assertions switched off, the behavior is undefined and will most
-             likely yield segmentation violation.
-
-    @tparam ContiguousContainer container type with contiguous storage
-    @param[in] c  container to read from
-    @param[in] cb  a parser callback function of type @ref parser_callback_t
-    which is used to control the deserialization by filtering unwanted values
-    (optional)
-
-    @return result of the deserialization
-
-    @throw parse_error.101 in case of an unexpected token
-    @throw parse_error.102 if to_unicode fails or surrogate error
-    @throw parse_error.103 if to_unicode fails
-
-    @complexity Linear in the length of the input. The parser is a predictive
-    LL(1) parser. The complexity can be higher if the parser callback function
-    @a cb has a super-linear complexity.
-
-    @note A UTF-8 byte order mark is silently ignored.
-
-    @liveexample{The example below demonstrates the `parse()` function reading
-    from a contiguous container.,parse__contiguouscontainer__parser_callback_t}
-
-    @since version 2.0.3
-    */
-    template<class ContiguousContainer, typename std::enable_if<
-                 not std::is_pointer<ContiguousContainer>::value and
-                 std::is_base_of<
-                     std::random_access_iterator_tag,
-                     typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value
-                 , int>::type = 0>
-    static basic_json parse(const ContiguousContainer& c,
-                            const parser_callback_t cb = nullptr)
-    {
-        // delegate the call to the iterator-range parse overload
-        return parse(std::begin(c), std::end(c), cb);
-    }
-
-    template<class ContiguousContainer, typename std::enable_if<
-                 not std::is_pointer<ContiguousContainer>::value and
-                 std::is_base_of<
-                     std::random_access_iterator_tag,
-                     typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value
-                 , int>::type = 0>
-    static bool accept(const ContiguousContainer& c)
-    {
-        // delegate the call to the iterator-range accept overload
-        return accept(std::begin(c), std::end(c));
+        return parser(detail::input_adapter(first, last)).accept(true);
     }
 
     /*!
     @brief deserialize from stream
     @deprecated This stream operator is deprecated and will be removed in a
                 future version of the library. Please use
-                @ref std::istream& operator>>(std::istream&, basic_json&)
+                @ref operator>>(std::istream&, basic_json&)
                 instead; that is, replace calls like `j << i;` with `i >> j;`.
     */
     JSON_DEPRECATED
     friend std::istream& operator<<(basic_json& j, std::istream& i)
     {
-        j = parser(input_adapter::create(i)).parse(false);
-        return i;
+        return operator>>(i, j);
     }
 
     /*!
@@ -7768,7 +13039,6 @@ class basic_json
     @throw parse_error.101 in case of an unexpected token
     @throw parse_error.102 if to_unicode fails or surrogate error
     @throw parse_error.103 if to_unicode fails
-    @throw parse_error.111 if input stream is in a bad state
 
     @complexity Linear in the length of the input. The parser is a predictive
     LL(1) parser.
@@ -7785,7 +13055,7 @@ class basic_json
     */
     friend std::istream& operator>>(std::istream& i, basic_json& j)
     {
-        j = parser(input_adapter::create(i)).parse(false);
+        parser(detail::input_adapter(i)).parse(false, j);
         return i;
     }
 
@@ -7801,16 +13071,31 @@ class basic_json
     Returns the type name as string to be used in error messages - usually to
     indicate that a function was called on a wrong JSON type.
 
-    @return basically a string representation of a the @a m_type member
+    @return a string representation of a the @a m_type member:
+            Value type  | return value
+            ----------- | -------------
+            null        | `"null"`
+            boolean     | `"boolean"`
+            string      | `"string"`
+            number      | `"number"` (for all number types)
+            object      | `"object"`
+            array       | `"array"`
+            discarded   | `"discarded"`
+
+    @exceptionsafety No-throw guarantee: this function never throws exceptions.
 
     @complexity Constant.
 
     @liveexample{The following code exemplifies `type_name()` for all JSON
     types.,type_name}
 
-    @since version 1.0.0, public since 2.1.0
+    @sa @ref type() -- return the type of the JSON value
+    @sa @ref operator value_t() -- return the type of the JSON value (implicit) 
+
+    @since version 1.0.0, public since 2.1.0, `const char*` and `noexcept`
+    since 3.0.0
     */
-    std::string type_name() const
+    const char* type_name() const noexcept
     {
         {
             switch (m_type)
@@ -7845,1239 +13130,6 @@ class basic_json
     /// the value of the current element
     json_value m_value = {};
 
-
-  private:
-    ///////////////
-    // iterators //
-    ///////////////
-
-    /*!
-    @brief an iterator for primitive JSON types
-
-    This class models an iterator for primitive JSON types (boolean, number,
-    string). It's only purpose is to allow the iterator/const_iterator classes
-    to "iterate" over primitive values. Internally, the iterator is modeled by
-    a `difference_type` variable. Value begin_value (`0`) models the begin,
-    end_value (`1`) models past the end.
-    */
-    class primitive_iterator_t
-    {
-      public:
-
-        difference_type get_value() const noexcept
-        {
-            return m_it;
-        }
-        /// set iterator to a defined beginning
-        void set_begin() noexcept
-        {
-            m_it = begin_value;
-        }
-
-        /// set iterator to a defined past the end
-        void set_end() noexcept
-        {
-            m_it = end_value;
-        }
-
-        /// return whether the iterator can be dereferenced
-        constexpr bool is_begin() const noexcept
-        {
-            return (m_it == begin_value);
-        }
-
-        /// return whether the iterator is at end
-        constexpr bool is_end() const noexcept
-        {
-            return (m_it == end_value);
-        }
-
-        friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-        {
-            return lhs.m_it == rhs.m_it;
-        }
-
-        friend constexpr bool operator!=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-        {
-            return !(lhs == rhs);
-        }
-
-        friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-        {
-            return lhs.m_it < rhs.m_it;
-        }
-
-        friend constexpr bool operator<=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-        {
-            return lhs.m_it <= rhs.m_it;
-        }
-
-        friend constexpr bool operator>(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-        {
-            return lhs.m_it > rhs.m_it;
-        }
-
-        friend constexpr bool operator>=(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-        {
-            return lhs.m_it >= rhs.m_it;
-        }
-
-        primitive_iterator_t operator+(difference_type i)
-        {
-            auto result = *this;
-            result += i;
-            return result;
-        }
-
-        friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept
-        {
-            return lhs.m_it - rhs.m_it;
-        }
-
-        friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it)
-        {
-            return os << it.m_it;
-        }
-
-        primitive_iterator_t& operator++()
-        {
-            ++m_it;
-            return *this;
-        }
-
-        primitive_iterator_t operator++(int)
-        {
-            auto result = *this;
-            m_it++;
-            return result;
-        }
-
-        primitive_iterator_t& operator--()
-        {
-            --m_it;
-            return *this;
-        }
-
-        primitive_iterator_t operator--(int)
-        {
-            auto result = *this;
-            m_it--;
-            return result;
-        }
-
-        primitive_iterator_t& operator+=(difference_type n)
-        {
-            m_it += n;
-            return *this;
-        }
-
-        primitive_iterator_t& operator-=(difference_type n)
-        {
-            m_it -= n;
-            return *this;
-        }
-
-      private:
-        static constexpr difference_type begin_value = 0;
-        static constexpr difference_type end_value = begin_value + 1;
-
-        /// iterator as signed integer type
-        difference_type m_it = std::numeric_limits<std::ptrdiff_t>::denorm_min();
-    };
-
-    /*!
-    @brief an iterator value
-
-    @note This structure could easily be a union, but MSVC currently does not
-    allow unions members with complex constructors, see
-    https://github.com/nlohmann/json/pull/105.
-    */
-    struct internal_iterator
-    {
-        /// iterator for JSON objects
-        typename object_t::iterator object_iterator;
-        /// iterator for JSON arrays
-        typename array_t::iterator array_iterator;
-        /// generic iterator for all other types
-        primitive_iterator_t primitive_iterator;
-
-        /// create an uninitialized internal_iterator
-        internal_iterator() noexcept
-            : object_iterator(), array_iterator(), primitive_iterator()
-        {}
-    };
-
-    /// proxy class for the iterator_wrapper functions
-    template<typename IteratorType>
-    class iteration_proxy
-    {
-      private:
-        /// helper class for iteration
-        class iteration_proxy_internal
-        {
-          private:
-            /// the iterator
-            IteratorType anchor;
-            /// an index for arrays (used to create key names)
-            size_t array_index = 0;
-
-          public:
-            explicit iteration_proxy_internal(IteratorType it) noexcept
-                : anchor(it)
-            {}
-
-            /// dereference operator (needed for range-based for)
-            iteration_proxy_internal& operator*()
-            {
-                return *this;
-            }
-
-            /// increment operator (needed for range-based for)
-            iteration_proxy_internal& operator++()
-            {
-                ++anchor;
-                ++array_index;
-
-                return *this;
-            }
-
-            /// inequality operator (needed for range-based for)
-            bool operator!= (const iteration_proxy_internal& o) const
-            {
-                return anchor != o.anchor;
-            }
-
-            /// return key of the iterator
-            typename basic_json::string_t key() const
-            {
-                assert(anchor.m_object != nullptr);
-
-                switch (anchor.m_object->type())
-                {
-                    // use integer array index as key
-                    case value_t::array:
-                    {
-                        return std::to_string(array_index);
-                    }
-
-                    // use key from the object
-                    case value_t::object:
-                    {
-                        return anchor.key();
-                    }
-
-                    // use an empty key for all primitive types
-                    default:
-                    {
-                        return "";
-                    }
-                }
-            }
-
-            /// return value of the iterator
-            typename IteratorType::reference value() const
-            {
-                return anchor.value();
-            }
-        };
-
-        /// the container to iterate
-        typename IteratorType::reference container;
-
-      public:
-        /// construct iteration proxy from a container
-        explicit iteration_proxy(typename IteratorType::reference cont)
-            : container(cont)
-        {}
-
-        /// return iterator begin (needed for range-based for)
-        iteration_proxy_internal begin() noexcept
-        {
-            return iteration_proxy_internal(container.begin());
-        }
-
-        /// return iterator end (needed for range-based for)
-        iteration_proxy_internal end() noexcept
-        {
-            return iteration_proxy_internal(container.end());
-        }
-    };
-
-  public:
-    /*!
-    @brief a template for a random access iterator for the @ref basic_json class
-
-    This class implements a both iterators (iterator and const_iterator) for the
-    @ref basic_json class.
-
-    @note An iterator is called *initialized* when a pointer to a JSON value
-          has been set (e.g., by a constructor or a copy assignment). If the
-          iterator is default-constructed, it is *uninitialized* and most
-          methods are undefined. **The library uses assertions to detect calls
-          on uninitialized iterators.**
-
-    @requirement The class satisfies the following concept requirements:
-    - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator):
-      The iterator that can be moved to point (forward and backward) to any
-      element in constant time.
-
-    @since version 1.0.0, simplified in version 2.0.9
-    */
-    template<typename U>
-    class iter_impl : public std::iterator<std::random_access_iterator_tag, U>
-    {
-        /// allow basic_json to access private members
-        friend class basic_json;
-
-        // make sure U is basic_json or const basic_json
-        static_assert(std::is_same<U, basic_json>::value
-                      or std::is_same<U, const basic_json>::value,
-                      "iter_impl only accepts (const) basic_json");
-
-      public:
-        /// the type of the values when the iterator is dereferenced
-        using value_type = typename basic_json::value_type;
-        /// a type to represent differences between iterators
-        using difference_type = typename basic_json::difference_type;
-        /// defines a pointer to the type iterated over (value_type)
-        using pointer = typename std::conditional<std::is_const<U>::value,
-              typename basic_json::const_pointer,
-              typename basic_json::pointer>::type;
-        /// defines a reference to the type iterated over (value_type)
-        using reference = typename std::conditional<std::is_const<U>::value,
-              typename basic_json::const_reference,
-              typename basic_json::reference>::type;
-        /// the category of the iterator
-        using iterator_category = std::bidirectional_iterator_tag;
-
-        /// default constructor
-        iter_impl() = default;
-
-        /*!
-        @brief constructor for a given JSON instance
-        @param[in] object  pointer to a JSON object for this iterator
-        @pre object != nullptr
-        @post The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        explicit iter_impl(pointer object) noexcept
-            : m_object(object)
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    m_it.object_iterator = typename object_t::iterator();
-                    break;
-                }
-
-                case basic_json::value_t::array:
-                {
-                    m_it.array_iterator = typename array_t::iterator();
-                    break;
-                }
-
-                default:
-                {
-                    m_it.primitive_iterator = primitive_iterator_t();
-                    break;
-                }
-            }
-        }
-
-        /*!
-        @note The conventional copy constructor and copy assignment are
-              implicitly defined.
-              Combined with the following converting constructor and assigment,
-              they support: copy from iterator to iterator,
-                            copy from const iterator to const iterator,
-                            and conversion from iterator to const iterator.
-              However conversion from const iterator to iterator is not defined.
-        */
-
-        /*!
-        @brief converting constructor
-        @param[in] other  non-const iterator to copy from
-        @note It is not checked whether @a other is initialized.
-        */
-        iter_impl(const iter_impl<basic_json>& other) noexcept
-            : m_object(other.m_object), m_it(other.m_it)
-        {}
-
-        /*!
-        @brief converting assignment
-        @param[in,out] other  non-const iterator to copy from
-        @return const/non-const iterator
-        @note It is not checked whether @a other is initialized.
-        */
-        iter_impl& operator=(const iter_impl<basic_json>& other) noexcept
-        {
-            m_object = other.m_object;
-            m_it = other.m_it;
-            return *this;
-        }
-
-      private:
-        /*!
-        @brief set the iterator to the first value
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        void set_begin() noexcept
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    m_it.object_iterator = m_object->m_value.object->begin();
-                    break;
-                }
-
-                case basic_json::value_t::array:
-                {
-                    m_it.array_iterator = m_object->m_value.array->begin();
-                    break;
-                }
-
-                case basic_json::value_t::null:
-                {
-                    // set to end so begin()==end() is true: null is empty
-                    m_it.primitive_iterator.set_end();
-                    break;
-                }
-
-                default:
-                {
-                    m_it.primitive_iterator.set_begin();
-                    break;
-                }
-            }
-        }
-
-        /*!
-        @brief set the iterator past the last value
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        void set_end() noexcept
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    m_it.object_iterator = m_object->m_value.object->end();
-                    break;
-                }
-
-                case basic_json::value_t::array:
-                {
-                    m_it.array_iterator = m_object->m_value.array->end();
-                    break;
-                }
-
-                default:
-                {
-                    m_it.primitive_iterator.set_end();
-                    break;
-                }
-            }
-        }
-
-      public:
-        /*!
-        @brief return a reference to the value pointed to by the iterator
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        reference operator*() const
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    assert(m_it.object_iterator != m_object->m_value.object->end());
-                    return m_it.object_iterator->second;
-                }
-
-                case basic_json::value_t::array:
-                {
-                    assert(m_it.array_iterator != m_object->m_value.array->end());
-                    return *m_it.array_iterator;
-                }
-
-                case basic_json::value_t::null:
-                {
-                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-                }
-
-                default:
-                {
-                    if (m_it.primitive_iterator.is_begin())
-                    {
-                        return *m_object;
-                    }
-
-                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-                }
-            }
-        }
-
-        /*!
-        @brief dereference the iterator
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        pointer operator->() const
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    assert(m_it.object_iterator != m_object->m_value.object->end());
-                    return &(m_it.object_iterator->second);
-                }
-
-                case basic_json::value_t::array:
-                {
-                    assert(m_it.array_iterator != m_object->m_value.array->end());
-                    return &*m_it.array_iterator;
-                }
-
-                default:
-                {
-                    if (m_it.primitive_iterator.is_begin())
-                    {
-                        return m_object;
-                    }
-
-                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-                }
-            }
-        }
-
-        /*!
-        @brief post-increment (it++)
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        iter_impl operator++(int)
-        {
-            auto result = *this;
-            ++(*this);
-            return result;
-        }
-
-        /*!
-        @brief pre-increment (++it)
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        iter_impl& operator++()
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    std::advance(m_it.object_iterator, 1);
-                    break;
-                }
-
-                case basic_json::value_t::array:
-                {
-                    std::advance(m_it.array_iterator, 1);
-                    break;
-                }
-
-                default:
-                {
-                    ++m_it.primitive_iterator;
-                    break;
-                }
-            }
-
-            return *this;
-        }
-
-        /*!
-        @brief post-decrement (it--)
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        iter_impl operator--(int)
-        {
-            auto result = *this;
-            --(*this);
-            return result;
-        }
-
-        /*!
-        @brief pre-decrement (--it)
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        iter_impl& operator--()
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    std::advance(m_it.object_iterator, -1);
-                    break;
-                }
-
-                case basic_json::value_t::array:
-                {
-                    std::advance(m_it.array_iterator, -1);
-                    break;
-                }
-
-                default:
-                {
-                    --m_it.primitive_iterator;
-                    break;
-                }
-            }
-
-            return *this;
-        }
-
-        /*!
-        @brief  comparison: equal
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        bool operator==(const iter_impl& other) const
-        {
-            // if objects are not the same, the comparison is undefined
-            if (m_object != other.m_object)
-            {
-                JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
-            }
-
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    return (m_it.object_iterator == other.m_it.object_iterator);
-                }
-
-                case basic_json::value_t::array:
-                {
-                    return (m_it.array_iterator == other.m_it.array_iterator);
-                }
-
-                default:
-                {
-                    return (m_it.primitive_iterator == other.m_it.primitive_iterator);
-                }
-            }
-        }
-
-        /*!
-        @brief  comparison: not equal
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        bool operator!=(const iter_impl& other) const
-        {
-            return not operator==(other);
-        }
-
-        /*!
-        @brief  comparison: smaller
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        bool operator<(const iter_impl& other) const
-        {
-            // if objects are not the same, the comparison is undefined
-            if (m_object != other.m_object)
-            {
-                JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers"));
-            }
-
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators"));
-                }
-
-                case basic_json::value_t::array:
-                {
-                    return (m_it.array_iterator < other.m_it.array_iterator);
-                }
-
-                default:
-                {
-                    return (m_it.primitive_iterator < other.m_it.primitive_iterator);
-                }
-            }
-        }
-
-        /*!
-        @brief  comparison: less than or equal
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        bool operator<=(const iter_impl& other) const
-        {
-            return not other.operator < (*this);
-        }
-
-        /*!
-        @brief  comparison: greater than
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        bool operator>(const iter_impl& other) const
-        {
-            return not operator<=(other);
-        }
-
-        /*!
-        @brief  comparison: greater than or equal
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        bool operator>=(const iter_impl& other) const
-        {
-            return not operator<(other);
-        }
-
-        /*!
-        @brief  add to iterator
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        iter_impl& operator+=(difference_type i)
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
-                }
-
-                case basic_json::value_t::array:
-                {
-                    std::advance(m_it.array_iterator, i);
-                    break;
-                }
-
-                default:
-                {
-                    m_it.primitive_iterator += i;
-                    break;
-                }
-            }
-
-            return *this;
-        }
-
-        /*!
-        @brief  subtract from iterator
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        iter_impl& operator-=(difference_type i)
-        {
-            return operator+=(-i);
-        }
-
-        /*!
-        @brief  add to iterator
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        iter_impl operator+(difference_type i) const
-        {
-            auto result = *this;
-            result += i;
-            return result;
-        }
-
-        /*!
-        @brief  addition of distance and iterator
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        friend iter_impl operator+(difference_type i, const iter_impl& it)
-        {
-            auto result = it;
-            result += i;
-            return result;
-        }
-
-        /*!
-        @brief  subtract from iterator
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        iter_impl operator-(difference_type i) const
-        {
-            auto result = *this;
-            result -= i;
-            return result;
-        }
-
-        /*!
-        @brief  return difference
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        difference_type operator-(const iter_impl& other) const
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators"));
-                }
-
-                case basic_json::value_t::array:
-                {
-                    return m_it.array_iterator - other.m_it.array_iterator;
-                }
-
-                default:
-                {
-                    return m_it.primitive_iterator - other.m_it.primitive_iterator;
-                }
-            }
-        }
-
-        /*!
-        @brief  access to successor
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        reference operator[](difference_type n) const
-        {
-            assert(m_object != nullptr);
-
-            switch (m_object->m_type)
-            {
-                case basic_json::value_t::object:
-                {
-                    JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators"));
-                }
-
-                case basic_json::value_t::array:
-                {
-                    return *std::next(m_it.array_iterator, n);
-                }
-
-                case basic_json::value_t::null:
-                {
-                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-                }
-
-                default:
-                {
-                    if (m_it.primitive_iterator.get_value() == -n)
-                    {
-                        return *m_object;
-                    }
-
-                    JSON_THROW(invalid_iterator::create(214, "cannot get value"));
-                }
-            }
-        }
-
-        /*!
-        @brief  return the key of an object iterator
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        typename object_t::key_type key() const
-        {
-            assert(m_object != nullptr);
-
-            if (m_object->is_object())
-            {
-                return m_it.object_iterator->first;
-            }
-
-            JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators"));
-        }
-
-        /*!
-        @brief  return the value of an iterator
-        @pre The iterator is initialized; i.e. `m_object != nullptr`.
-        */
-        reference value() const
-        {
-            return operator*();
-        }
-
-      private:
-        /// associated JSON instance
-        pointer m_object = nullptr;
-        /// the actual iterator of the associated instance
-        struct internal_iterator m_it = internal_iterator();
-    };
-
-    /*!
-    @brief a template for a reverse iterator class
-
-    @tparam Base the base iterator type to reverse. Valid types are @ref
-    iterator (to create @ref reverse_iterator) and @ref const_iterator (to
-    create @ref const_reverse_iterator).
-
-    @requirement The class satisfies the following concept requirements:
-    - [RandomAccessIterator](http://en.cppreference.com/w/cpp/concept/RandomAccessIterator):
-      The iterator that can be moved to point (forward and backward) to any
-      element in constant time.
-    - [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator):
-      It is possible to write to the pointed-to element (only if @a Base is
-      @ref iterator).
-
-    @since version 1.0.0
-    */
-    template<typename Base>
-    class json_reverse_iterator : public std::reverse_iterator<Base>
-    {
-      public:
-        /// shortcut to the reverse iterator adaptor
-        using base_iterator = std::reverse_iterator<Base>;
-        /// the reference type for the pointed-to element
-        using reference = typename Base::reference;
-
-        /// create reverse iterator from iterator
-        json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept
-            : base_iterator(it)
-        {}
-
-        /// create reverse iterator from base class
-        json_reverse_iterator(const base_iterator& it) noexcept
-            : base_iterator(it)
-        {}
-
-        /// post-increment (it++)
-        json_reverse_iterator operator++(int)
-        {
-            return static_cast<json_reverse_iterator>(base_iterator::operator++(1));
-        }
-
-        /// pre-increment (++it)
-        json_reverse_iterator& operator++()
-        {
-            return static_cast<json_reverse_iterator&>(base_iterator::operator++());
-        }
-
-        /// post-decrement (it--)
-        json_reverse_iterator operator--(int)
-        {
-            return static_cast<json_reverse_iterator>(base_iterator::operator--(1));
-        }
-
-        /// pre-decrement (--it)
-        json_reverse_iterator& operator--()
-        {
-            return static_cast<json_reverse_iterator&>(base_iterator::operator--());
-        }
-
-        /// add to iterator
-        json_reverse_iterator& operator+=(difference_type i)
-        {
-            return static_cast<json_reverse_iterator&>(base_iterator::operator+=(i));
-        }
-
-        /// add to iterator
-        json_reverse_iterator operator+(difference_type i) const
-        {
-            return static_cast<json_reverse_iterator>(base_iterator::operator+(i));
-        }
-
-        /// subtract from iterator
-        json_reverse_iterator operator-(difference_type i) const
-        {
-            return static_cast<json_reverse_iterator>(base_iterator::operator-(i));
-        }
-
-        /// return difference
-        difference_type operator-(const json_reverse_iterator& other) const
-        {
-            return base_iterator(*this) - base_iterator(other);
-        }
-
-        /// access to successor
-        reference operator[](difference_type n) const
-        {
-            return *(this->operator+(n));
-        }
-
-        /// return the key of an object iterator
-        typename object_t::key_type key() const
-        {
-            auto it = --this->base();
-            return it.key();
-        }
-
-        /// return the value of an iterator
-        reference value() const
-        {
-            auto it = --this->base();
-            return it.operator * ();
-        }
-    };
-
-
-  private:
-    ////////////////////
-    // input adapters //
-    ////////////////////
-
-    /// abstract input adapter interface
-    class input_adapter
-    {
-      public:
-        virtual int get_character() = 0;
-        virtual std::string read(size_t offset, size_t length) = 0;
-        virtual ~input_adapter() {}
-
-        // native support
-
-        /// input adapter for input stream
-        static std::shared_ptr<input_adapter> create(std::istream& i)
-        {
-            return std::shared_ptr<input_adapter>(new cached_input_stream_adapter<16384>(i));
-        }
-
-        /// input adapter for input stream
-        static std::shared_ptr<input_adapter> create(std::istream&& i)
-        {
-            return std::shared_ptr<input_adapter>(new cached_input_stream_adapter<16384>(i));
-        }
-
-        /// input adapter for buffer
-        static std::shared_ptr<input_adapter> create(const char* b, size_t l)
-        {
-            return std::shared_ptr<input_adapter>(new input_buffer_adapter(b, l));
-        }
-
-        // derived support
-
-        /// input adapter for string literal
-        template<typename CharT, typename std::enable_if<
-                     std::is_pointer<CharT>::value and
-                     std::is_integral<typename std::remove_pointer<CharT>::type>::value and
-                     sizeof(typename std::remove_pointer<CharT>::type) == 1, int>::type = 0>
-        static std::shared_ptr<input_adapter> create(CharT b)
-        {
-            return create(reinterpret_cast<const char*>(b),
-                          std::strlen(reinterpret_cast<const char*>(b)));
-        }
-
-        /// input adapter for iterator range with contiguous storage
-        template<class IteratorType, typename std::enable_if<
-                     std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value
-                     , int>::type
-                 = 0>
-        static std::shared_ptr<input_adapter> create(IteratorType first, IteratorType last)
-        {
-            // assertion to check that the iterator range is indeed contiguous,
-            // see http://stackoverflow.com/a/35008842/266378 for more discussion
-            assert(std::accumulate(first, last, std::pair<bool, int>(true, 0),
-                                   [&first](std::pair<bool, int> res, decltype(*first) val)
-            {
-                res.first &= (val == *(std::next(std::addressof(*first), res.second++)));
-                return res;
-            }).first);
-
-            // assertion to check that each element is 1 byte long
-            static_assert(sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1,
-                          "each element in the iterator range must have the size of 1 byte");
-
-            return create(reinterpret_cast<const char*>(&(*first)),
-                          static_cast<size_t>(std::distance(first, last)));
-        }
-
-        /// input adapter for array
-        template<class T, std::size_t N>
-        static std::shared_ptr<input_adapter> create(T (&array)[N])
-        {
-            // delegate the call to the iterator-range overload
-            return create(std::begin(array), std::end(array));
-        }
-
-        /// input adapter for contiguous container
-        template<class ContiguousContainer, typename std::enable_if<
-                     not std::is_pointer<ContiguousContainer>::value and
-                     std::is_base_of<
-                         std::random_access_iterator_tag,
-                         typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value
-                     , int>::type = 0>
-        static std::shared_ptr<input_adapter> create(const ContiguousContainer& c)
-        {
-            // delegate the call to the iterator-range overload
-            return create(std::begin(c), std::end(c));
-        }
-    };
-
-    /// a type to simplify interfaces
-    using input_adapter_t = std::shared_ptr<input_adapter>;
-
-    /// input adapter for cached stream input
-    template<std::size_t N>
-    class cached_input_stream_adapter : public input_adapter
-    {
-      public:
-        cached_input_stream_adapter(std::istream& i)
-            : is(i), start_position(is.tellg())
-        {
-            // immediately abort if stream is erroneous
-            if (JSON_UNLIKELY(i.fail()))
-            {
-                JSON_THROW(parse_error::create(111, 0, "bad input stream"));
-            }
-
-            fill_buffer();
-
-            // skip byte order mark
-            if (fill_size >= 3 and buffer[0] == '\xEF' and buffer[1] == '\xBB' and buffer[2] == '\xBF')
-            {
-                buffer_pos += 3;
-                processed_chars += 3;
-            }
-        }
-
-        ~cached_input_stream_adapter() override
-        {
-            // clear stream flags
-            is.clear();
-            // We initially read a lot of characters into the buffer, and we
-            // may not have processed all of them. Therefore, we need to
-            // "rewind" the stream after the last processed char.
-            is.seekg(start_position);
-            is.ignore(static_cast<std::streamsize>(processed_chars));
-            // clear stream flags
-            is.clear();
-        }
-
-        int get_character() override
-        {
-            // check if refilling is necessary and possible
-            if (buffer_pos == fill_size and not eof)
-            {
-                fill_buffer();
-
-                // check and remember that filling did not yield new input
-                if (fill_size == 0)
-                {
-                    eof = true;
-                    return std::char_traits<char>::eof();
-                }
-
-                // the buffer is ready
-                buffer_pos = 0;
-            }
-
-            ++processed_chars;
-            assert(buffer_pos < buffer.size());
-            return buffer[buffer_pos++] & 0xFF;
-        }
-
-        std::string read(size_t offset, size_t length) override
-        {
-            // create buffer
-            std::string result(length, '\0');
-
-            // save stream position
-            const auto current_pos = is.tellg();
-            // save stream flags
-            const auto flags = is.rdstate();
-
-            // clear stream flags
-            is.clear();
-            // set stream position
-            is.seekg(static_cast<std::streamoff>(offset));
-            // read bytes
-            is.read(&result[0], static_cast<std::streamsize>(length));
-
-            // reset stream position
-            is.seekg(current_pos);
-            // reset stream flags
-            is.setstate(flags);
-
-            return result;
-        }
-
-      private:
-        void fill_buffer()
-        {
-            // fill
-            is.read(buffer.data(), static_cast<std::streamsize>(buffer.size()));
-            // store number of bytes in the buffer
-            fill_size = static_cast<size_t>(is.gcount());
-        }
-
-        /// the associated input stream
-        std::istream& is;
-
-        /// chars returned via get_character()
-        size_t processed_chars = 0;
-        /// chars processed in the current buffer
-        size_t buffer_pos = 0;
-
-        /// whether stream reached eof
-        bool eof = false;
-        /// how many chars have been copied to the buffer by last (re)fill
-        size_t fill_size = 0;
-
-        /// position of the stream when we started
-        const std::streampos start_position;
-
-        /// internal buffer
-        std::array<char, N> buffer{{}};
-    };
-
-    /// input adapter for buffer input
-    class input_buffer_adapter : public input_adapter
-    {
-      public:
-        input_buffer_adapter(const char* b, size_t l)
-            : input_adapter(), cursor(b), limit(b + l), start(b)
-        {
-            // skip byte order mark
-            if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF')
-            {
-                cursor += 3;
-            }
-        }
-
-        // delete because of pointer members
-        input_buffer_adapter(const input_buffer_adapter&) = delete;
-        input_buffer_adapter& operator=(input_buffer_adapter&) = delete;
-
-        int get_character() noexcept override
-        {
-            if (JSON_LIKELY(cursor < limit))
-            {
-                return *(cursor++) & 0xFF;
-            }
-            else
-            {
-                return std::char_traits<char>::eof();
-            }
-        }
-
-        std::string read(size_t offset, size_t length) override
-        {
-            // avoid reading too many characters
-            const size_t max_length = static_cast<size_t>(limit - start);
-            return std::string(start + offset, (std::min)(length, max_length - offset));
-        }
-
-      private:
-        /// pointer to the current character
-        const char* cursor;
-        /// pointer past the last character
-        const char* limit;
-        /// pointer to the first character
-        const char* start;
-    };
-
     //////////////////////////////////////////
     // binary serialization/deserialization //
     //////////////////////////////////////////
@@ -9085,1678 +13137,6 @@ class basic_json
     /// @name binary serialization/deserialization support
     /// @{
 
-  private:
-    /*!
-    @brief deserialization of CBOR and MessagePack values
-    */
-    class binary_reader
-    {
-      public:
-        /*!
-        @brief create a binary reader
-
-        @param[in] adapter  input adapter to read from
-        */
-        explicit binary_reader(input_adapter_t adapter)
-            : ia(adapter), is_little_endian(little_endianess())
-        {
-            assert(ia);
-        }
-
-        /*!
-        @brief create a JSON value from CBOR input
-
-        @param[in] get_char  whether a new character should be retrieved from
-                             the input (true, default) or whether the last
-                             read character should be considered instead
-
-        @return JSON value created from CBOR input
-
-        @throw parse_error.110 if input ended unexpectedly
-        @throw parse_error.112 if unsupported byte was read
-        */
-        basic_json parse_cbor(const bool get_char = true)
-        {
-            switch (get_char ? get() : current)
-            {
-                // EOF
-                case std::char_traits<char>::eof():
-                {
-                    JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
-                }
-
-                // Integer 0x00..0x17 (0..23)
-                case 0x00:
-                case 0x01:
-                case 0x02:
-                case 0x03:
-                case 0x04:
-                case 0x05:
-                case 0x06:
-                case 0x07:
-                case 0x08:
-                case 0x09:
-                case 0x0a:
-                case 0x0b:
-                case 0x0c:
-                case 0x0d:
-                case 0x0e:
-                case 0x0f:
-                case 0x10:
-                case 0x11:
-                case 0x12:
-                case 0x13:
-                case 0x14:
-                case 0x15:
-                case 0x16:
-                case 0x17:
-                {
-                    return static_cast<number_unsigned_t>(current);
-                }
-
-                case 0x18: // Unsigned integer (one-byte uint8_t follows)
-                {
-                    return get_number<uint8_t>();
-                }
-
-                case 0x19: // Unsigned integer (two-byte uint16_t follows)
-                {
-                    return get_number<uint16_t>();
-                }
-
-                case 0x1a: // Unsigned integer (four-byte uint32_t follows)
-                {
-                    return get_number<uint32_t>();
-                }
-
-                case 0x1b: // Unsigned integer (eight-byte uint64_t follows)
-                {
-                    return get_number<uint64_t>();
-                }
-
-                // Negative integer -1-0x00..-1-0x17 (-1..-24)
-                case 0x20:
-                case 0x21:
-                case 0x22:
-                case 0x23:
-                case 0x24:
-                case 0x25:
-                case 0x26:
-                case 0x27:
-                case 0x28:
-                case 0x29:
-                case 0x2a:
-                case 0x2b:
-                case 0x2c:
-                case 0x2d:
-                case 0x2e:
-                case 0x2f:
-                case 0x30:
-                case 0x31:
-                case 0x32:
-                case 0x33:
-                case 0x34:
-                case 0x35:
-                case 0x36:
-                case 0x37:
-                {
-                    return static_cast<int8_t>(0x20 - 1 - current);
-                }
-
-                case 0x38: // Negative integer (one-byte uint8_t follows)
-                {
-                    // must be uint8_t !
-                    return static_cast<number_integer_t>(-1) - get_number<uint8_t>();
-                }
-
-                case 0x39: // Negative integer -1-n (two-byte uint16_t follows)
-                {
-                    return static_cast<number_integer_t>(-1) - get_number<uint16_t>();
-                }
-
-                case 0x3a: // Negative integer -1-n (four-byte uint32_t follows)
-                {
-                    return static_cast<number_integer_t>(-1) - get_number<uint32_t>();
-                }
-
-                case 0x3b: // Negative integer -1-n (eight-byte uint64_t follows)
-                {
-                    return static_cast<number_integer_t>(-1) - static_cast<number_integer_t>(get_number<uint64_t>());
-                }
-
-                // UTF-8 string (0x00..0x17 bytes follow)
-                case 0x60:
-                case 0x61:
-                case 0x62:
-                case 0x63:
-                case 0x64:
-                case 0x65:
-                case 0x66:
-                case 0x67:
-                case 0x68:
-                case 0x69:
-                case 0x6a:
-                case 0x6b:
-                case 0x6c:
-                case 0x6d:
-                case 0x6e:
-                case 0x6f:
-                case 0x70:
-                case 0x71:
-                case 0x72:
-                case 0x73:
-                case 0x74:
-                case 0x75:
-                case 0x76:
-                case 0x77:
-                case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
-                case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
-                case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
-                case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
-                case 0x7f: // UTF-8 string (indefinite length)
-                {
-                    return get_cbor_string();
-                }
-
-                // array (0x00..0x17 data items follow)
-                case 0x80:
-                case 0x81:
-                case 0x82:
-                case 0x83:
-                case 0x84:
-                case 0x85:
-                case 0x86:
-                case 0x87:
-                case 0x88:
-                case 0x89:
-                case 0x8a:
-                case 0x8b:
-                case 0x8c:
-                case 0x8d:
-                case 0x8e:
-                case 0x8f:
-                case 0x90:
-                case 0x91:
-                case 0x92:
-                case 0x93:
-                case 0x94:
-                case 0x95:
-                case 0x96:
-                case 0x97:
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(current & 0x1f);
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(parse_cbor());
-                    }
-                    return result;
-                }
-
-                case 0x98: // array (one-byte uint8_t for n follows)
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(get_number<uint8_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(parse_cbor());
-                    }
-                    return result;
-                }
-
-                case 0x99: // array (two-byte uint16_t for n follow)
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(get_number<uint16_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(parse_cbor());
-                    }
-                    return result;
-                }
-
-                case 0x9a: // array (four-byte uint32_t for n follow)
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(get_number<uint32_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(parse_cbor());
-                    }
-                    return result;
-                }
-
-                case 0x9b: // array (eight-byte uint64_t for n follow)
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(get_number<uint64_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(parse_cbor());
-                    }
-                    return result;
-                }
-
-                case 0x9f: // array (indefinite length)
-                {
-                    basic_json result = value_t::array;
-                    while (get() != 0xff)
-                    {
-                        result.push_back(parse_cbor(false));
-                    }
-                    return result;
-                }
-
-                // map (0x00..0x17 pairs of data items follow)
-                case 0xa0:
-                case 0xa1:
-                case 0xa2:
-                case 0xa3:
-                case 0xa4:
-                case 0xa5:
-                case 0xa6:
-                case 0xa7:
-                case 0xa8:
-                case 0xa9:
-                case 0xaa:
-                case 0xab:
-                case 0xac:
-                case 0xad:
-                case 0xae:
-                case 0xaf:
-                case 0xb0:
-                case 0xb1:
-                case 0xb2:
-                case 0xb3:
-                case 0xb4:
-                case 0xb5:
-                case 0xb6:
-                case 0xb7:
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(current & 0x1f);
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        get();
-                        auto key = get_cbor_string();
-                        result[key] = parse_cbor();
-                    }
-                    return result;
-                }
-
-                case 0xb8: // map (one-byte uint8_t for n follows)
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(get_number<uint8_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        get();
-                        auto key = get_cbor_string();
-                        result[key] = parse_cbor();
-                    }
-                    return result;
-                }
-
-                case 0xb9: // map (two-byte uint16_t for n follow)
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(get_number<uint16_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        get();
-                        auto key = get_cbor_string();
-                        result[key] = parse_cbor();
-                    }
-                    return result;
-                }
-
-                case 0xba: // map (four-byte uint32_t for n follow)
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(get_number<uint32_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        get();
-                        auto key = get_cbor_string();
-                        result[key] = parse_cbor();
-                    }
-                    return result;
-                }
-
-                case 0xbb: // map (eight-byte uint64_t for n follow)
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(get_number<uint64_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        get();
-                        auto key = get_cbor_string();
-                        result[key] = parse_cbor();
-                    }
-                    return result;
-                }
-
-                case 0xbf: // map (indefinite length)
-                {
-                    basic_json result = value_t::object;
-                    while (get() != 0xff)
-                    {
-                        auto key = get_cbor_string();
-                        result[key] = parse_cbor();
-                    }
-                    return result;
-                }
-
-                case 0xf4: // false
-                {
-                    return false;
-                }
-
-                case 0xf5: // true
-                {
-                    return true;
-                }
-
-                case 0xf6: // null
-                {
-                    return value_t::null;
-                }
-
-                case 0xf9: // Half-Precision Float (two-byte IEEE 754)
-                {
-                    const int byte1 = get();
-                    check_eof();
-                    const int byte2 = get();
-                    check_eof();
-
-                    // code from RFC 7049, Appendix D, Figure 3:
-                    // As half-precision floating-point numbers were only added
-                    // to IEEE 754 in 2008, today's programming platforms often
-                    // still only have limited support for them. It is very
-                    // easy to include at least decoding support for them even
-                    // without such support. An example of a small decoder for
-                    // half-precision floating-point numbers in the C language
-                    // is shown in Fig. 3.
-                    const int half = (byte1 << 8) + byte2;
-                    const int exp = (half >> 10) & 0x1f;
-                    const int mant = half & 0x3ff;
-                    double val;
-                    if (exp == 0)
-                    {
-                        val = std::ldexp(mant, -24);
-                    }
-                    else if (exp != 31)
-                    {
-                        val = std::ldexp(mant + 1024, exp - 25);
-                    }
-                    else
-                    {
-                        val = mant == 0
-                              ? std::numeric_limits<double>::infinity()
-                              : std::numeric_limits<double>::quiet_NaN();
-                    }
-                    return (half & 0x8000) != 0 ? -val : val;
-                }
-
-                case 0xfa: // Single-Precision Float (four-byte IEEE 754)
-                {
-                    return get_number<float>();
-                }
-
-                case 0xfb: // Double-Precision Float (eight-byte IEEE 754)
-                {
-                    return get_number<double>();
-                }
-
-                default: // anything else (0xFF is handled inside the other types)
-                {
-                    std::stringstream ss;
-                    ss << std::setw(2) << std::setfill('0') << std::hex << current;
-                    JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str()));
-                }
-            }
-        }
-
-        /*!
-        @brief create a JSON value from MessagePack input
-
-        @return JSON value created from MessagePack input
-
-        @throw parse_error.110 if input ended unexpectedly
-        @throw parse_error.112 if unsupported byte was read
-        */
-        basic_json parse_msgpack()
-        {
-            switch (get())
-            {
-                // EOF
-                case std::char_traits<char>::eof():
-                {
-                    JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
-                }
-
-                // positive fixint
-                case 0x00:
-                case 0x01:
-                case 0x02:
-                case 0x03:
-                case 0x04:
-                case 0x05:
-                case 0x06:
-                case 0x07:
-                case 0x08:
-                case 0x09:
-                case 0x0a:
-                case 0x0b:
-                case 0x0c:
-                case 0x0d:
-                case 0x0e:
-                case 0x0f:
-                case 0x10:
-                case 0x11:
-                case 0x12:
-                case 0x13:
-                case 0x14:
-                case 0x15:
-                case 0x16:
-                case 0x17:
-                case 0x18:
-                case 0x19:
-                case 0x1a:
-                case 0x1b:
-                case 0x1c:
-                case 0x1d:
-                case 0x1e:
-                case 0x1f:
-                case 0x20:
-                case 0x21:
-                case 0x22:
-                case 0x23:
-                case 0x24:
-                case 0x25:
-                case 0x26:
-                case 0x27:
-                case 0x28:
-                case 0x29:
-                case 0x2a:
-                case 0x2b:
-                case 0x2c:
-                case 0x2d:
-                case 0x2e:
-                case 0x2f:
-                case 0x30:
-                case 0x31:
-                case 0x32:
-                case 0x33:
-                case 0x34:
-                case 0x35:
-                case 0x36:
-                case 0x37:
-                case 0x38:
-                case 0x39:
-                case 0x3a:
-                case 0x3b:
-                case 0x3c:
-                case 0x3d:
-                case 0x3e:
-                case 0x3f:
-                case 0x40:
-                case 0x41:
-                case 0x42:
-                case 0x43:
-                case 0x44:
-                case 0x45:
-                case 0x46:
-                case 0x47:
-                case 0x48:
-                case 0x49:
-                case 0x4a:
-                case 0x4b:
-                case 0x4c:
-                case 0x4d:
-                case 0x4e:
-                case 0x4f:
-                case 0x50:
-                case 0x51:
-                case 0x52:
-                case 0x53:
-                case 0x54:
-                case 0x55:
-                case 0x56:
-                case 0x57:
-                case 0x58:
-                case 0x59:
-                case 0x5a:
-                case 0x5b:
-                case 0x5c:
-                case 0x5d:
-                case 0x5e:
-                case 0x5f:
-                case 0x60:
-                case 0x61:
-                case 0x62:
-                case 0x63:
-                case 0x64:
-                case 0x65:
-                case 0x66:
-                case 0x67:
-                case 0x68:
-                case 0x69:
-                case 0x6a:
-                case 0x6b:
-                case 0x6c:
-                case 0x6d:
-                case 0x6e:
-                case 0x6f:
-                case 0x70:
-                case 0x71:
-                case 0x72:
-                case 0x73:
-                case 0x74:
-                case 0x75:
-                case 0x76:
-                case 0x77:
-                case 0x78:
-                case 0x79:
-                case 0x7a:
-                case 0x7b:
-                case 0x7c:
-                case 0x7d:
-                case 0x7e:
-                case 0x7f:
-                {
-                    return static_cast<number_unsigned_t>(current);
-                }
-
-                // fixmap
-                case 0x80:
-                case 0x81:
-                case 0x82:
-                case 0x83:
-                case 0x84:
-                case 0x85:
-                case 0x86:
-                case 0x87:
-                case 0x88:
-                case 0x89:
-                case 0x8a:
-                case 0x8b:
-                case 0x8c:
-                case 0x8d:
-                case 0x8e:
-                case 0x8f:
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(current & 0x0f);
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        get();
-                        auto key = get_msgpack_string();
-                        result[key] = parse_msgpack();
-                    }
-                    return result;
-                }
-
-                // fixarray
-                case 0x90:
-                case 0x91:
-                case 0x92:
-                case 0x93:
-                case 0x94:
-                case 0x95:
-                case 0x96:
-                case 0x97:
-                case 0x98:
-                case 0x99:
-                case 0x9a:
-                case 0x9b:
-                case 0x9c:
-                case 0x9d:
-                case 0x9e:
-                case 0x9f:
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(current & 0x0f);
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(parse_msgpack());
-                    }
-                    return result;
-                }
-
-                // fixstr
-                case 0xa0:
-                case 0xa1:
-                case 0xa2:
-                case 0xa3:
-                case 0xa4:
-                case 0xa5:
-                case 0xa6:
-                case 0xa7:
-                case 0xa8:
-                case 0xa9:
-                case 0xaa:
-                case 0xab:
-                case 0xac:
-                case 0xad:
-                case 0xae:
-                case 0xaf:
-                case 0xb0:
-                case 0xb1:
-                case 0xb2:
-                case 0xb3:
-                case 0xb4:
-                case 0xb5:
-                case 0xb6:
-                case 0xb7:
-                case 0xb8:
-                case 0xb9:
-                case 0xba:
-                case 0xbb:
-                case 0xbc:
-                case 0xbd:
-                case 0xbe:
-                case 0xbf:
-                {
-                    return get_msgpack_string();
-                }
-
-                case 0xc0: // nil
-                {
-                    return value_t::null;
-                }
-
-                case 0xc2: // false
-                {
-                    return false;
-                }
-
-                case 0xc3: // true
-                {
-                    return true;
-                }
-
-                case 0xca: // float 32
-                {
-                    return get_number<float>();
-                }
-
-                case 0xcb: // float 64
-                {
-                    return get_number<double>();
-                }
-
-                case 0xcc: // uint 8
-                {
-                    return get_number<uint8_t>();
-                }
-
-                case 0xcd: // uint 16
-                {
-                    return get_number<uint16_t>();
-                }
-
-                case 0xce: // uint 32
-                {
-                    return get_number<uint32_t>();
-                }
-
-                case 0xcf: // uint 64
-                {
-                    return get_number<uint64_t>();
-                }
-
-                case 0xd0: // int 8
-                {
-                    return get_number<int8_t>();
-                }
-
-                case 0xd1: // int 16
-                {
-                    return get_number<int16_t>();
-                }
-
-                case 0xd2: // int 32
-                {
-                    return get_number<int32_t>();
-                }
-
-                case 0xd3: // int 64
-                {
-                    return get_number<int64_t>();
-                }
-
-                case 0xd9: // str 8
-                case 0xda: // str 16
-                case 0xdb: // str 32
-                {
-                    return get_msgpack_string();
-                }
-
-                case 0xdc: // array 16
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(get_number<uint16_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(parse_msgpack());
-                    }
-                    return result;
-                }
-
-                case 0xdd: // array 32
-                {
-                    basic_json result = value_t::array;
-                    const auto len = static_cast<size_t>(get_number<uint32_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        result.push_back(parse_msgpack());
-                    }
-                    return result;
-                }
-
-                case 0xde: // map 16
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(get_number<uint16_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        get();
-                        auto key = get_msgpack_string();
-                        result[key] = parse_msgpack();
-                    }
-                    return result;
-                }
-
-                case 0xdf: // map 32
-                {
-                    basic_json result = value_t::object;
-                    const auto len = static_cast<size_t>(get_number<uint32_t>());
-                    for (size_t i = 0; i < len; ++i)
-                    {
-                        get();
-                        auto key = get_msgpack_string();
-                        result[key] = parse_msgpack();
-                    }
-                    return result;
-                }
-
-                // positive fixint
-                case 0xe0:
-                case 0xe1:
-                case 0xe2:
-                case 0xe3:
-                case 0xe4:
-                case 0xe5:
-                case 0xe6:
-                case 0xe7:
-                case 0xe8:
-                case 0xe9:
-                case 0xea:
-                case 0xeb:
-                case 0xec:
-                case 0xed:
-                case 0xee:
-                case 0xef:
-                case 0xf0:
-                case 0xf1:
-                case 0xf2:
-                case 0xf3:
-                case 0xf4:
-                case 0xf5:
-                case 0xf6:
-                case 0xf7:
-                case 0xf8:
-                case 0xf9:
-                case 0xfa:
-                case 0xfb:
-                case 0xfc:
-                case 0xfd:
-                case 0xfe:
-                case 0xff:
-                {
-                    return static_cast<int8_t>(current);
-                }
-
-                default: // anything else
-                {
-                    std::stringstream ss;
-                    ss << std::setw(2) << std::setfill('0') << std::hex << current;
-                    JSON_THROW(parse_error::create(112, chars_read, "error reading MessagePack; last byte: 0x" + ss.str()));
-                }
-            }
-        }
-
-        /*!
-        @brief determine system byte order
-
-        @return true iff system's byte order is little endian
-
-        @note from http://stackoverflow.com/a/1001328/266378
-        */
-        static bool little_endianess() noexcept
-        {
-            int num = 1;
-            return (*reinterpret_cast<char*>(&num) == 1);
-        }
-
-      private:
-        /*!
-        @brief get next character from the input
-
-        This function provides the interface to the used input adapter. It does
-        not throw in case the input reached EOF, but returns
-        `std::char_traits<char>::eof()` in that case.
-
-        @return character read from the input
-        */
-        int get()
-        {
-            ++chars_read;
-            return (current = ia->get_character());
-        }
-
-        /*
-        @brief read a number from the input
-
-        @tparam T the type of the number
-
-        @return number of type @a T
-
-        @note This function needs to respect the system's endianess, because
-              bytes in CBOR and MessagePack are stored in network order (big
-              endian) and therefore need reordering on little endian systems.
-
-        @throw parse_error.110 if input has less than `sizeof(T)` bytes
-        */
-        template<typename T>
-        T get_number()
-        {
-            // step 1: read input into array with system's byte order
-            std::array<uint8_t, sizeof(T)> vec;
-            for (size_t i = 0; i < sizeof(T); ++i)
-            {
-                get();
-                check_eof();
-
-                // reverse byte order prior to conversion if necessary
-                if (is_little_endian)
-                {
-                    vec[sizeof(T) - i - 1] = static_cast<uint8_t>(current);
-                }
-                else
-                {
-                    vec[i] = static_cast<uint8_t>(current);  // LCOV_EXCL_LINE
-                }
-            }
-
-            // step 2: convert array into number of type T and return
-            T result;
-            std::memcpy(&result, vec.data(), sizeof(T));
-            return result;
-        }
-
-        /*!
-        @brief create a string by reading characters from the input
-
-        @param[in] len number of bytes to read
-
-        @return string created by reading @a len bytes
-
-        @throw parse_error.110 if input has less than @a len bytes
-        */
-        std::string get_string(const size_t len)
-        {
-            std::string result;
-            for (size_t i = 0; i < len; ++i)
-            {
-                get();
-                check_eof();
-                result.append(1, static_cast<char>(current));
-            }
-            return result;
-        }
-
-        /*!
-        @brief reads a CBOR string
-
-        This function first reads starting bytes to determine the expected
-        string length and then copies this number of bytes into a string.
-        Additionally, CBOR's strings with indefinite lengths are supported.
-
-        @return string
-
-        @throw parse_error.110 if input ended
-        @throw parse_error.113 if an unexpexted byte is read
-        */
-        std::string get_cbor_string()
-        {
-            check_eof();
-
-            switch (current)
-            {
-                // UTF-8 string (0x00..0x17 bytes follow)
-                case 0x60:
-                case 0x61:
-                case 0x62:
-                case 0x63:
-                case 0x64:
-                case 0x65:
-                case 0x66:
-                case 0x67:
-                case 0x68:
-                case 0x69:
-                case 0x6a:
-                case 0x6b:
-                case 0x6c:
-                case 0x6d:
-                case 0x6e:
-                case 0x6f:
-                case 0x70:
-                case 0x71:
-                case 0x72:
-                case 0x73:
-                case 0x74:
-                case 0x75:
-                case 0x76:
-                case 0x77:
-                {
-                    const auto len = static_cast<size_t>(current & 0x1f);
-                    return get_string(len);
-                }
-
-                case 0x78: // UTF-8 string (one-byte uint8_t for n follows)
-                {
-                    const auto len = static_cast<size_t>(get_number<uint8_t>());
-                    return get_string(len);
-                }
-
-                case 0x79: // UTF-8 string (two-byte uint16_t for n follow)
-                {
-                    const auto len = static_cast<size_t>(get_number<uint16_t>());
-                    return get_string(len);
-                }
-
-                case 0x7a: // UTF-8 string (four-byte uint32_t for n follow)
-                {
-                    const auto len = static_cast<size_t>(get_number<uint32_t>());
-                    return get_string(len);
-                }
-
-                case 0x7b: // UTF-8 string (eight-byte uint64_t for n follow)
-                {
-                    const auto len = static_cast<size_t>(get_number<uint64_t>());
-                    return get_string(len);
-                }
-
-                case 0x7f: // UTF-8 string (indefinite length)
-                {
-                    std::string result;
-                    while (get() != 0xff)
-                    {
-                        check_eof();
-                        result.append(1, static_cast<char>(current));
-                    }
-                    return result;
-                }
-
-                default:
-                {
-                    std::stringstream ss;
-                    ss << std::setw(2) << std::setfill('0') << std::hex << current;
-                    JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str()));
-                }
-            }
-        }
-
-        /*!
-        @brief reads a MessagePack string
-
-        This function first reads starting bytes to determine the expected
-        string length and then copies this number of bytes into a string.
-
-        @return string
-
-        @throw parse_error.110 if input ended
-        @throw parse_error.113 if an unexpexted byte is read
-        */
-        std::string get_msgpack_string()
-        {
-            check_eof();
-
-            switch (current)
-            {
-                // fixstr
-                case 0xa0:
-                case 0xa1:
-                case 0xa2:
-                case 0xa3:
-                case 0xa4:
-                case 0xa5:
-                case 0xa6:
-                case 0xa7:
-                case 0xa8:
-                case 0xa9:
-                case 0xaa:
-                case 0xab:
-                case 0xac:
-                case 0xad:
-                case 0xae:
-                case 0xaf:
-                case 0xb0:
-                case 0xb1:
-                case 0xb2:
-                case 0xb3:
-                case 0xb4:
-                case 0xb5:
-                case 0xb6:
-                case 0xb7:
-                case 0xb8:
-                case 0xb9:
-                case 0xba:
-                case 0xbb:
-                case 0xbc:
-                case 0xbd:
-                case 0xbe:
-                case 0xbf:
-                {
-                    const auto len = static_cast<size_t>(current & 0x1f);
-                    return get_string(len);
-                }
-
-                case 0xd9: // str 8
-                {
-                    const auto len = static_cast<size_t>(get_number<uint8_t>());
-                    return get_string(len);
-                }
-
-                case 0xda: // str 16
-                {
-                    const auto len = static_cast<size_t>(get_number<uint16_t>());
-                    return get_string(len);
-                }
-
-                case 0xdb: // str 32
-                {
-                    const auto len = static_cast<size_t>(get_number<uint32_t>());
-                    return get_string(len);
-                }
-
-                default:
-                {
-                    std::stringstream ss;
-                    ss << std::setw(2) << std::setfill('0') << std::hex << current;
-                    JSON_THROW(parse_error::create(113, chars_read, "expected a MessagePack string; last byte: 0x" + ss.str()));
-                }
-            }
-        }
-
-        /*!
-        @brief check if input ended
-        @throw parse_error.110 if input ended
-        */
-        void check_eof() const
-        {
-            if (JSON_UNLIKELY(current == std::char_traits<char>::eof()))
-            {
-                JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input"));
-            }
-        }
-
-      private:
-        /// input adapter
-        input_adapter_t ia = nullptr;
-
-        /// the current character
-        int current = std::char_traits<char>::eof();
-
-        /// the number of characters read
-        size_t chars_read = 0;
-
-        /// whether we can assume little endianess
-        const bool is_little_endian = true;
-    };
-
-    /*!
-    @brief serialization to CBOR and MessagePack values
-    */
-    class binary_writer
-    {
-      public:
-        /*!
-        @brief create a binary writer
-
-        @param[in] adapter  output adapter to write to
-        */
-        explicit binary_writer(output_adapter_t<uint8_t> adapter)
-            : is_little_endian(binary_reader::little_endianess()), oa(adapter)
-        {
-            assert(oa);
-        }
-
-        /*!
-        @brief[in] j  JSON value to serialize
-        */
-        void write_cbor(const basic_json& j)
-        {
-            switch (j.type())
-            {
-                case value_t::null:
-                {
-                    oa->write_character(0xf6);
-                    break;
-                }
-
-                case value_t::boolean:
-                {
-                    oa->write_character(j.m_value.boolean ? 0xf5 : 0xf4);
-                    break;
-                }
-
-                case value_t::number_integer:
-                {
-                    if (j.m_value.number_integer >= 0)
-                    {
-                        // CBOR does not differentiate between positive signed
-                        // integers and unsigned integers. Therefore, we used the
-                        // code from the value_t::number_unsigned case here.
-                        if (j.m_value.number_integer <= 0x17)
-                        {
-                            write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)())
-                        {
-                            oa->write_character(0x18);
-                            write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)())
-                        {
-                            oa->write_character(0x19);
-                            write_number(static_cast<uint16_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)())
-                        {
-                            oa->write_character(0x1a);
-                            write_number(static_cast<uint32_t>(j.m_value.number_integer));
-                        }
-                        else
-                        {
-                            oa->write_character(0x1b);
-                            write_number(static_cast<uint64_t>(j.m_value.number_integer));
-                        }
-                    }
-                    else
-                    {
-                        // The conversions below encode the sign in the first
-                        // byte, and the value is converted to a positive number.
-                        const auto positive_number = -1 - j.m_value.number_integer;
-                        if (j.m_value.number_integer >= -24)
-                        {
-                            write_number(static_cast<uint8_t>(0x20 + positive_number));
-                        }
-                        else if (positive_number <= (std::numeric_limits<uint8_t>::max)())
-                        {
-                            oa->write_character(0x38);
-                            write_number(static_cast<uint8_t>(positive_number));
-                        }
-                        else if (positive_number <= (std::numeric_limits<uint16_t>::max)())
-                        {
-                            oa->write_character(0x39);
-                            write_number(static_cast<uint16_t>(positive_number));
-                        }
-                        else if (positive_number <= (std::numeric_limits<uint32_t>::max)())
-                        {
-                            oa->write_character(0x3a);
-                            write_number(static_cast<uint32_t>(positive_number));
-                        }
-                        else
-                        {
-                            oa->write_character(0x3b);
-                            write_number(static_cast<uint64_t>(positive_number));
-                        }
-                    }
-                    break;
-                }
-
-                case value_t::number_unsigned:
-                {
-                    if (j.m_value.number_unsigned <= 0x17)
-                    {
-                        write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
-                    {
-                        oa->write_character(0x18);
-                        write_number(static_cast<uint8_t>(j.m_value.number_unsigned));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
-                    {
-                        oa->write_character(0x19);
-                        write_number(static_cast<uint16_t>(j.m_value.number_unsigned));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
-                    {
-                        oa->write_character(0x1a);
-                        write_number(static_cast<uint32_t>(j.m_value.number_unsigned));
-                    }
-                    else
-                    {
-                        oa->write_character(0x1b);
-                        write_number(static_cast<uint64_t>(j.m_value.number_unsigned));
-                    }
-                    break;
-                }
-
-                case value_t::number_float:
-                {
-                    // Double-Precision Float
-                    oa->write_character(0xfb);
-                    write_number(j.m_value.number_float);
-                    break;
-                }
-
-                case value_t::string:
-                {
-                    // step 1: write control byte and the string length
-                    const auto N = j.m_value.string->size();
-                    if (N <= 0x17)
-                    {
-                        write_number(static_cast<uint8_t>(0x60 + N));
-                    }
-                    else if (N <= 0xff)
-                    {
-                        oa->write_character(0x78);
-                        write_number(static_cast<uint8_t>(N));
-                    }
-                    else if (N <= 0xffff)
-                    {
-                        oa->write_character(0x79);
-                        write_number(static_cast<uint16_t>(N));
-                    }
-                    else if (N <= 0xffffffff)
-                    {
-                        oa->write_character(0x7a);
-                        write_number(static_cast<uint32_t>(N));
-                    }
-                    // LCOV_EXCL_START
-                    else if (N <= 0xffffffffffffffff)
-                    {
-                        oa->write_character(0x7b);
-                        write_number(static_cast<uint64_t>(N));
-                    }
-                    // LCOV_EXCL_STOP
-
-                    // step 2: write the string
-                    oa->write_characters(reinterpret_cast<const uint8_t*>(j.m_value.string->c_str()),
-                                         j.m_value.string->size());
-                    break;
-                }
-
-                case value_t::array:
-                {
-                    // step 1: write control byte and the array size
-                    const auto N = j.m_value.array->size();
-                    if (N <= 0x17)
-                    {
-                        write_number(static_cast<uint8_t>(0x80 + N));
-                    }
-                    else if (N <= 0xff)
-                    {
-                        oa->write_character(0x98);
-                        write_number(static_cast<uint8_t>(N));
-                    }
-                    else if (N <= 0xffff)
-                    {
-                        oa->write_character(0x99);
-                        write_number(static_cast<uint16_t>(N));
-                    }
-                    else if (N <= 0xffffffff)
-                    {
-                        oa->write_character(0x9a);
-                        write_number(static_cast<uint32_t>(N));
-                    }
-                    // LCOV_EXCL_START
-                    else if (N <= 0xffffffffffffffff)
-                    {
-                        oa->write_character(0x9b);
-                        write_number(static_cast<uint64_t>(N));
-                    }
-                    // LCOV_EXCL_STOP
-
-                    // step 2: write each element
-                    for (const auto& el : *j.m_value.array)
-                    {
-                        write_cbor(el);
-                    }
-                    break;
-                }
-
-                case value_t::object:
-                {
-                    // step 1: write control byte and the object size
-                    const auto N = j.m_value.object->size();
-                    if (N <= 0x17)
-                    {
-                        write_number(static_cast<uint8_t>(0xa0 + N));
-                    }
-                    else if (N <= 0xff)
-                    {
-                        oa->write_character(0xb8);
-                        write_number(static_cast<uint8_t>(N));
-                    }
-                    else if (N <= 0xffff)
-                    {
-                        oa->write_character(0xb9);
-                        write_number(static_cast<uint16_t>(N));
-                    }
-                    else if (N <= 0xffffffff)
-                    {
-                        oa->write_character(0xba);
-                        write_number(static_cast<uint32_t>(N));
-                    }
-                    // LCOV_EXCL_START
-                    else if (N <= 0xffffffffffffffff)
-                    {
-                        oa->write_character(0xbb);
-                        write_number(static_cast<uint64_t>(N));
-                    }
-                    // LCOV_EXCL_STOP
-
-                    // step 2: write each element
-                    for (const auto& el : *j.m_value.object)
-                    {
-                        write_cbor(el.first);
-                        write_cbor(el.second);
-                    }
-                    break;
-                }
-
-                default:
-                {
-                    break;
-                }
-            }
-        }
-
-        /*!
-        @brief[in] j  JSON value to serialize
-        */
-        void write_msgpack(const basic_json& j)
-        {
-            switch (j.type())
-            {
-                case value_t::null:
-                {
-                    // nil
-                    oa->write_character(0xc0);
-                    break;
-                }
-
-                case value_t::boolean:
-                {
-                    // true and false
-                    oa->write_character(j.m_value.boolean ? 0xc3 : 0xc2);
-                    break;
-                }
-
-                case value_t::number_integer:
-                {
-                    if (j.m_value.number_integer >= 0)
-                    {
-                        // MessagePack does not differentiate between positive
-                        // signed integers and unsigned integers. Therefore, we
-                        // used the code from the value_t::number_unsigned case
-                        // here.
-                        if (j.m_value.number_unsigned < 128)
-                        {
-                            // positive fixnum
-                            write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
-                        {
-                            // uint 8
-                            oa->write_character(0xcc);
-                            write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
-                        {
-                            // uint 16
-                            oa->write_character(0xcd);
-                            write_number(static_cast<uint16_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
-                        {
-                            // uint 32
-                            oa->write_character(0xce);
-                            write_number(static_cast<uint32_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
-                        {
-                            // uint 64
-                            oa->write_character(0xcf);
-                            write_number(static_cast<uint64_t>(j.m_value.number_integer));
-                        }
-                    }
-                    else
-                    {
-                        if (j.m_value.number_integer >= -32)
-                        {
-                            // negative fixnum
-                            write_number(static_cast<int8_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)())
-                        {
-                            // int 8
-                            oa->write_character(0xd0);
-                            write_number(static_cast<int8_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)())
-                        {
-                            // int 16
-                            oa->write_character(0xd1);
-                            write_number(static_cast<int16_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)())
-                        {
-                            // int 32
-                            oa->write_character(0xd2);
-                            write_number(static_cast<int32_t>(j.m_value.number_integer));
-                        }
-                        else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)())
-                        {
-                            // int 64
-                            oa->write_character(0xd3);
-                            write_number(static_cast<int64_t>(j.m_value.number_integer));
-                        }
-                    }
-                    break;
-                }
-
-                case value_t::number_unsigned:
-                {
-                    if (j.m_value.number_unsigned < 128)
-                    {
-                        // positive fixnum
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)())
-                    {
-                        // uint 8
-                        oa->write_character(0xcc);
-                        write_number(static_cast<uint8_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)())
-                    {
-                        // uint 16
-                        oa->write_character(0xcd);
-                        write_number(static_cast<uint16_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)())
-                    {
-                        // uint 32
-                        oa->write_character(0xce);
-                        write_number(static_cast<uint32_t>(j.m_value.number_integer));
-                    }
-                    else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)())
-                    {
-                        // uint 64
-                        oa->write_character(0xcf);
-                        write_number(static_cast<uint64_t>(j.m_value.number_integer));
-                    }
-                    break;
-                }
-
-                case value_t::number_float:
-                {
-                    // float 64
-                    oa->write_character(0xcb);
-                    write_number(j.m_value.number_float);
-                    break;
-                }
-
-                case value_t::string:
-                {
-                    // step 1: write control byte and the string length
-                    const auto N = j.m_value.string->size();
-                    if (N <= 31)
-                    {
-                        // fixstr
-                        write_number(static_cast<uint8_t>(0xa0 | N));
-                    }
-                    else if (N <= 255)
-                    {
-                        // str 8
-                        oa->write_character(0xd9);
-                        write_number(static_cast<uint8_t>(N));
-                    }
-                    else if (N <= 65535)
-                    {
-                        // str 16
-                        oa->write_character(0xda);
-                        write_number(static_cast<uint16_t>(N));
-                    }
-                    else if (N <= 4294967295)
-                    {
-                        // str 32
-                        oa->write_character(0xdb);
-                        write_number(static_cast<uint32_t>(N));
-                    }
-
-                    // step 2: write the string
-                    oa->write_characters(reinterpret_cast<const uint8_t*>(j.m_value.string->c_str()),
-                                         j.m_value.string->size());
-                    break;
-                }
-
-                case value_t::array:
-                {
-                    // step 1: write control byte and the array size
-                    const auto N = j.m_value.array->size();
-                    if (N <= 15)
-                    {
-                        // fixarray
-                        write_number(static_cast<uint8_t>(0x90 | N));
-                    }
-                    else if (N <= 0xffff)
-                    {
-                        // array 16
-                        oa->write_character(0xdc);
-                        write_number(static_cast<uint16_t>(N));
-                    }
-                    else if (N <= 0xffffffff)
-                    {
-                        // array 32
-                        oa->write_character(0xdd);
-                        write_number(static_cast<uint32_t>(N));
-                    }
-
-                    // step 2: write each element
-                    for (const auto& el : *j.m_value.array)
-                    {
-                        write_msgpack(el);
-                    }
-                    break;
-                }
-
-                case value_t::object:
-                {
-                    // step 1: write control byte and the object size
-                    const auto N = j.m_value.object->size();
-                    if (N <= 15)
-                    {
-                        // fixmap
-                        write_number(static_cast<uint8_t>(0x80 | (N & 0xf)));
-                    }
-                    else if (N <= 65535)
-                    {
-                        // map 16
-                        oa->write_character(0xde);
-                        write_number(static_cast<uint16_t>(N));
-                    }
-                    else if (N <= 4294967295)
-                    {
-                        // map 32
-                        oa->write_character(0xdf);
-                        write_number(static_cast<uint32_t>(N));
-                    }
-
-                    // step 2: write each element
-                    for (const auto& el : *j.m_value.object)
-                    {
-                        write_msgpack(el.first);
-                        write_msgpack(el.second);
-                    }
-                    break;
-                }
-
-                default:
-                {
-                    break;
-                }
-            }
-        }
-
-      private:
-        /*
-        @brief write a number to output input
-
-        @param[in] n number of type @a T
-        @tparam T the type of the number
-
-        @note This function needs to respect the system's endianess, because
-              bytes in CBOR and MessagePack are stored in network order (big
-              endian) and therefore need reordering on little endian systems.
-        */
-        template<typename T>
-        void write_number(T n)
-        {
-            // step 1: write number to array of length T
-            std::array<uint8_t, sizeof(T)> vec;
-            std::memcpy(vec.data(), &n, sizeof(T));
-
-            // step 2: write array to output (with possible reordering)
-            for (size_t i = 0; i < sizeof(T); ++i)
-            {
-                // reverse byte order prior to conversion if necessary
-                if (is_little_endian)
-                {
-                    oa->write_character(vec[sizeof(T) - i - 1]);
-                }
-                else
-                {
-                    oa->write_character(vec[i]);  // LCOV_EXCL_LINE
-                }
-            }
-        }
-
-      private:
-        /// whether we can assume little endianess
-        const bool is_little_endian = true;
-
-        /// the output
-        output_adapter_t<uint8_t> oa = nullptr;
-    };
-
   public:
     /*!
     @brief create a CBOR serialization of a given JSON value
@@ -10809,6 +13189,10 @@ class basic_json
     @note The mapping is **complete** in the sense that any JSON value type
           can be converted to a CBOR value.
 
+    @note If NaN or Infinity are stored inside a JSON number, they are
+          serialized properly. This behavior differs from the @ref dump()
+          function which serializes NaN or Infinity to `null`.
+
     @note The following CBOR types are not used in the conversion:
           - byte strings (0x40..0x5f)
           - UTF-8 strings terminated by "break" (0x7f)
@@ -10836,18 +13220,27 @@ class basic_json
     @sa http://cbor.io
     @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
         analogous deserialization
-    @sa @ref to_msgpack(const basic_json& for the related MessagePack format
+    @sa @ref to_msgpack(const basic_json&) for the related MessagePack format
 
     @since version 2.0.9
     */
     static std::vector<uint8_t> to_cbor(const basic_json& j)
     {
         std::vector<uint8_t> result;
-        binary_writer bw(output_adapter<uint8_t>::create(result));
-        bw.write_cbor(j);
+        to_cbor(j, result);
         return result;
     }
 
+    static void to_cbor(const basic_json& j, detail::output_adapter<uint8_t> o)
+    {
+        binary_writer<uint8_t>(o).write_cbor(j);
+    }
+
+    static void to_cbor(const basic_json& j, detail::output_adapter<char> o)
+    {
+        binary_writer<char>(o).write_cbor(j);
+    }
+
     /*!
     @brief create a MessagePack serialization of a given JSON value
 
@@ -10907,6 +13300,10 @@ class basic_json
     @note Any MessagePack output created @ref to_msgpack can be successfully
           parsed by @ref from_msgpack.
 
+    @note If NaN or Infinity are stored inside a JSON number, they are
+          serialized properly. This behavior differs from the @ref dump()
+          function which serializes NaN or Infinity to `null`.
+
     @param[in] j  JSON value to serialize
     @return MessagePack serialization as byte vector
 
@@ -10925,16 +13322,25 @@ class basic_json
     static std::vector<uint8_t> to_msgpack(const basic_json& j)
     {
         std::vector<uint8_t> result;
-        binary_writer bw(output_adapter<uint8_t>::create(result));
-        bw.write_msgpack(j);
+        to_msgpack(j, result);
         return result;
     }
 
-    /*!
-    @brief create a JSON value from a byte vector in CBOR format
+    static void to_msgpack(const basic_json& j, detail::output_adapter<uint8_t> o)
+    {
+        binary_writer<uint8_t>(o).write_msgpack(j);
+    }
 
-    Deserializes a given byte vector @a v to a JSON value using the CBOR
-    (Concise Binary Object Representation) serialization format.
+    static void to_msgpack(const basic_json& j, detail::output_adapter<char> o)
+    {
+        binary_writer<char>(o).write_msgpack(j);
+    }
+
+    /*!
+    @brief create a JSON value from an input in CBOR format
+
+    Deserializes a given input @a i to a JSON value using the CBOR (Concise
+    Binary Object Representation) serialization format.
 
     The library maps CBOR types to JSON value types as follows:
 
@@ -10996,39 +13402,51 @@ class basic_json
     @note Any CBOR output created @ref to_cbor can be successfully parsed by
           @ref from_cbor.
 
-    @param[in] v  a byte vector in CBOR format
-    @param[in] start_index the index to start reading from @a v (0 by default)
+    @param[in] i  an input in CBOR format convertible to an input adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
     @return deserialized JSON value
 
-    @throw parse_error.110 if the given vector ends prematurely
+    @throw parse_error.110 if the given input ends prematurely or the end of
+    file was not reached when @a strict was set to true
     @throw parse_error.112 if unsupported features from CBOR were
-    used in the given vector @a v or if the input is not valid CBOR
+    used in the given input @a v or if the input is not valid CBOR
     @throw parse_error.113 if a string was expected as map key, but not found
 
-    @complexity Linear in the size of the byte vector @a v.
+    @complexity Linear in the size of the input @a i.
 
     @liveexample{The example shows the deserialization of a byte vector in CBOR
     format to a JSON value.,from_cbor}
 
     @sa http://cbor.io
     @sa @ref to_cbor(const basic_json&) for the analogous serialization
-    @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the
+    @sa @ref from_msgpack(detail::input_adapter, const bool) for the
         related MessagePack format
 
-    @since version 2.0.9, parameter @a start_index since 2.1.1
+    @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
+           consume input adapters, removed start_index parameter, and added
+           @a strict parameter since 3.0.0
     */
-    static basic_json from_cbor(const std::vector<uint8_t>& v,
-                                const size_t start_index = 0)
+    static basic_json from_cbor(detail::input_adapter i,
+                                const bool strict = true)
     {
-        binary_reader br(input_adapter::create(v.begin() + static_cast<difference_type>(start_index), v.end()));
-        return br.parse_cbor();
+        return binary_reader(i).parse_cbor(strict);
     }
 
+    /*!
+    @copydoc from_cbor(detail::input_adapter, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_cbor(A1 && a1, A2 && a2, const bool strict = true)
+    {
+        return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_cbor(strict);
+    }
 
     /*!
-    @brief create a JSON value from a byte vector in MessagePack format
+    @brief create a JSON value from an input in MessagePack format
 
-    Deserializes a given byte vector @a v to a JSON value using the MessagePack
+    Deserializes a given input @a i to a JSON value using the MessagePack
     serialization format.
 
     The library maps MessagePack types to JSON value types as follows:
@@ -11071,2693 +13489,49 @@ class basic_json
     @note Any MessagePack output created @ref to_msgpack can be successfully
           parsed by @ref from_msgpack.
 
-    @param[in] v  a byte vector in MessagePack format
-    @param[in] start_index the index to start reading from @a v (0 by default)
-    @return deserialized JSON value
+    @param[in] i  an input in MessagePack format convertible to an input
+                  adapter
+    @param[in] strict  whether to expect the input to be consumed until EOF
+                       (true by default)
 
-    @throw parse_error.110 if the given vector ends prematurely
+    @throw parse_error.110 if the given input ends prematurely or the end of
+    file was not reached when @a strict was set to true
     @throw parse_error.112 if unsupported features from MessagePack were
-    used in the given vector @a v or if the input is not valid MessagePack
+    used in the given input @a i or if the input is not valid MessagePack
     @throw parse_error.113 if a string was expected as map key, but not found
 
-    @complexity Linear in the size of the byte vector @a v.
+    @complexity Linear in the size of the input @a i.
 
     @liveexample{The example shows the deserialization of a byte vector in
     MessagePack format to a JSON value.,from_msgpack}
 
     @sa http://msgpack.org
     @sa @ref to_msgpack(const basic_json&) for the analogous serialization
-    @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the
-        related CBOR format
+    @sa @ref from_cbor(detail::input_adapter, const bool) for the related CBOR
+        format
 
-    @since version 2.0.9, parameter @a start_index since 2.1.1
+    @since version 2.0.9; parameter @a start_index since 2.1.1; changed to
+           consume input adapters, removed start_index parameter, and added
+           @a strict parameter since 3.0.0
     */
-    static basic_json from_msgpack(const std::vector<uint8_t>& v,
-                                   const size_t start_index = 0)
+    static basic_json from_msgpack(detail::input_adapter i,
+                                   const bool strict = true)
     {
-        binary_reader br(input_adapter::create(v.begin() + static_cast<difference_type>(start_index), v.end()));
-        return br.parse_msgpack();
+        return binary_reader(i).parse_msgpack(strict);
+    }
+
+    /*!
+    @copydoc from_msgpack(detail::input_adapter, const bool)
+    */
+    template<typename A1, typename A2,
+             detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0>
+    static basic_json from_msgpack(A1 && a1, A2 && a2, const bool strict = true)
+    {
+        return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_msgpack(strict);
     }
 
     /// @}
 
-    //////////////////////
-    // lexer and parser //
-    //////////////////////
-
-  private:
-    /*!
-    @brief lexical analysis
-
-    This class organizes the lexical analysis during JSON deserialization.
-    */
-    class lexer
-    {
-      public:
-        /// token types for the parser
-        enum class token_type
-        {
-            uninitialized,   ///< indicating the scanner is uninitialized
-            literal_true,    ///< the `true` literal
-            literal_false,   ///< the `false` literal
-            literal_null,    ///< the `null` literal
-            value_string,    ///< a string -- use get_string() for actual value
-            value_unsigned,  ///< an unsigned integer -- use get_number_unsigned() for actual value
-            value_integer,   ///< a signed integer -- use get_number_integer() for actual value
-            value_float,     ///< an floating point number -- use get_number_float() for actual value
-            begin_array,     ///< the character for array begin `[`
-            begin_object,    ///< the character for object begin `{`
-            end_array,       ///< the character for array end `]`
-            end_object,      ///< the character for object end `}`
-            name_separator,  ///< the name separator `:`
-            value_separator, ///< the value separator `,`
-            parse_error,     ///< indicating a parse error
-            end_of_input,    ///< indicating the end of the input buffer
-            literal_or_value ///< a literal or the begin of a value (only for diagnostics)
-        };
-
-        /// return name of values of type token_type (only used for errors)
-        static const char* token_type_name(const token_type t) noexcept
-        {
-            switch (t)
-            {
-                case token_type::uninitialized:
-                    return "<uninitialized>";
-                case token_type::literal_true:
-                    return "true literal";
-                case token_type::literal_false:
-                    return "false literal";
-                case token_type::literal_null:
-                    return "null literal";
-                case token_type::value_string:
-                    return "string literal";
-                case lexer::token_type::value_unsigned:
-                case lexer::token_type::value_integer:
-                case lexer::token_type::value_float:
-                    return "number literal";
-                case token_type::begin_array:
-                    return "'['";
-                case token_type::begin_object:
-                    return "'{'";
-                case token_type::end_array:
-                    return "']'";
-                case token_type::end_object:
-                    return "'}'";
-                case token_type::name_separator:
-                    return "':'";
-                case token_type::value_separator:
-                    return "','";
-                case token_type::parse_error:
-                    return "<parse error>";
-                case token_type::end_of_input:
-                    return "end of input";
-                case token_type::literal_or_value:
-                    return "'[', '{', or a literal";
-                default:
-                {
-                    // catch non-enum values
-                    return "unknown token"; // LCOV_EXCL_LINE
-                }
-            }
-        }
-
-        explicit lexer(input_adapter_t adapter)
-            : ia(adapter), decimal_point_char(get_decimal_point())
-        {}
-
-        // delete because of pointer members
-        lexer(const lexer&) = delete;
-        lexer& operator=(lexer&) = delete;
-
-      private:
-        /////////////////////
-        // locales
-        /////////////////////
-
-        /// return the locale-dependent decimal point
-        static char get_decimal_point() noexcept
-        {
-            const auto loc = localeconv();
-            assert(loc != nullptr);
-            return (loc->decimal_point == nullptr) ? '.' : loc->decimal_point[0];
-        }
-
-        /////////////////////
-        // scan functions
-        /////////////////////
-
-        /*!
-        @brief get codepoint from 4 hex characters following `\u`
-
-        @return codepoint or -1 in case of an error (e.g. EOF or non-hex
-                character)
-        */
-        int get_codepoint()
-        {
-            // this function only makes sense after reading `\u`
-            assert(current == 'u');
-            int codepoint = 0;
-
-            // byte 1: \uXxxx
-            switch (get())
-            {
-                case '0':
-                    break;
-                case '1':
-                    codepoint += 0x1000;
-                    break;
-                case '2':
-                    codepoint += 0x2000;
-                    break;
-                case '3':
-                    codepoint += 0x3000;
-                    break;
-                case '4':
-                    codepoint += 0x4000;
-                    break;
-                case '5':
-                    codepoint += 0x5000;
-                    break;
-                case '6':
-                    codepoint += 0x6000;
-                    break;
-                case '7':
-                    codepoint += 0x7000;
-                    break;
-                case '8':
-                    codepoint += 0x8000;
-                    break;
-                case '9':
-                    codepoint += 0x9000;
-                    break;
-                case 'A':
-                case 'a':
-                    codepoint += 0xa000;
-                    break;
-                case 'B':
-                case 'b':
-                    codepoint += 0xb000;
-                    break;
-                case 'C':
-                case 'c':
-                    codepoint += 0xc000;
-                    break;
-                case 'D':
-                case 'd':
-                    codepoint += 0xd000;
-                    break;
-                case 'E':
-                case 'e':
-                    codepoint += 0xe000;
-                    break;
-                case 'F':
-                case 'f':
-                    codepoint += 0xf000;
-                    break;
-                default:
-                    return -1;
-            }
-
-            // byte 2: \uxXxx
-            switch (get())
-            {
-                case '0':
-                    break;
-                case '1':
-                    codepoint += 0x0100;
-                    break;
-                case '2':
-                    codepoint += 0x0200;
-                    break;
-                case '3':
-                    codepoint += 0x0300;
-                    break;
-                case '4':
-                    codepoint += 0x0400;
-                    break;
-                case '5':
-                    codepoint += 0x0500;
-                    break;
-                case '6':
-                    codepoint += 0x0600;
-                    break;
-                case '7':
-                    codepoint += 0x0700;
-                    break;
-                case '8':
-                    codepoint += 0x0800;
-                    break;
-                case '9':
-                    codepoint += 0x0900;
-                    break;
-                case 'A':
-                case 'a':
-                    codepoint += 0x0a00;
-                    break;
-                case 'B':
-                case 'b':
-                    codepoint += 0x0b00;
-                    break;
-                case 'C':
-                case 'c':
-                    codepoint += 0x0c00;
-                    break;
-                case 'D':
-                case 'd':
-                    codepoint += 0x0d00;
-                    break;
-                case 'E':
-                case 'e':
-                    codepoint += 0x0e00;
-                    break;
-                case 'F':
-                case 'f':
-                    codepoint += 0x0f00;
-                    break;
-                default:
-                    return -1;
-            }
-
-            // byte 3: \uxxXx
-            switch (get())
-            {
-                case '0':
-                    break;
-                case '1':
-                    codepoint += 0x0010;
-                    break;
-                case '2':
-                    codepoint += 0x0020;
-                    break;
-                case '3':
-                    codepoint += 0x0030;
-                    break;
-                case '4':
-                    codepoint += 0x0040;
-                    break;
-                case '5':
-                    codepoint += 0x0050;
-                    break;
-                case '6':
-                    codepoint += 0x0060;
-                    break;
-                case '7':
-                    codepoint += 0x0070;
-                    break;
-                case '8':
-                    codepoint += 0x0080;
-                    break;
-                case '9':
-                    codepoint += 0x0090;
-                    break;
-                case 'A':
-                case 'a':
-                    codepoint += 0x00a0;
-                    break;
-                case 'B':
-                case 'b':
-                    codepoint += 0x00b0;
-                    break;
-                case 'C':
-                case 'c':
-                    codepoint += 0x00c0;
-                    break;
-                case 'D':
-                case 'd':
-                    codepoint += 0x00d0;
-                    break;
-                case 'E':
-                case 'e':
-                    codepoint += 0x00e0;
-                    break;
-                case 'F':
-                case 'f':
-                    codepoint += 0x00f0;
-                    break;
-                default:
-                    return -1;
-            }
-
-            // byte 4: \uxxxX
-            switch (get())
-            {
-                case '0':
-                    break;
-                case '1':
-                    codepoint += 0x0001;
-                    break;
-                case '2':
-                    codepoint += 0x0002;
-                    break;
-                case '3':
-                    codepoint += 0x0003;
-                    break;
-                case '4':
-                    codepoint += 0x0004;
-                    break;
-                case '5':
-                    codepoint += 0x0005;
-                    break;
-                case '6':
-                    codepoint += 0x0006;
-                    break;
-                case '7':
-                    codepoint += 0x0007;
-                    break;
-                case '8':
-                    codepoint += 0x0008;
-                    break;
-                case '9':
-                    codepoint += 0x0009;
-                    break;
-                case 'A':
-                case 'a':
-                    codepoint += 0x000a;
-                    break;
-                case 'B':
-                case 'b':
-                    codepoint += 0x000b;
-                    break;
-                case 'C':
-                case 'c':
-                    codepoint += 0x000c;
-                    break;
-                case 'D':
-                case 'd':
-                    codepoint += 0x000d;
-                    break;
-                case 'E':
-                case 'e':
-                    codepoint += 0x000e;
-                    break;
-                case 'F':
-                case 'f':
-                    codepoint += 0x000f;
-                    break;
-                default:
-                    return -1;
-            }
-
-            return codepoint;
-        }
-
-        /*!
-        @brief scan a string literal
-
-        This function scans a string according to Sect. 7 of RFC 7159. While
-        scanning, bytes are escaped and copied into buffer yytext. Then the
-        function returns successfully, yytext is null-terminated and yylen
-        contains the number of bytes in the string.
-
-        @return token_type::value_string if string could be successfully
-                scanned, token_type::parse_error otherwise
-
-        @note In case of errors, variable error_message contains a textual
-              description.
-        */
-        token_type scan_string()
-        {
-            // reset yytext (ignore opening quote)
-            reset();
-
-            // we entered the function by reading an open quote
-            assert(current == '\"');
-
-            while (true)
-            {
-                // get next character
-                switch (get())
-                {
-                    // end of file while parsing string
-                    case std::char_traits<char>::eof():
-                    {
-                        error_message = "invalid string: missing closing quote";
-                        return token_type::parse_error;
-                    }
-
-                    // closing quote
-                    case '\"':
-                    {
-                        // terminate yytext
-                        add('\0');
-                        --yylen;
-                        return token_type::value_string;
-                    }
-
-                    // escapes
-                    case '\\':
-                    {
-                        switch (get())
-                        {
-                            // quotation mark
-                            case '\"':
-                                add('\"');
-                                break;
-                            // reverse solidus
-                            case '\\':
-                                add('\\');
-                                break;
-                            // solidus
-                            case '/':
-                                add('/');
-                                break;
-                            // backspace
-                            case 'b':
-                                add('\b');
-                                break;
-                            // form feed
-                            case 'f':
-                                add('\f');
-                                break;
-                            // line feed
-                            case 'n':
-                                add('\n');
-                                break;
-                            // carriage return
-                            case 'r':
-                                add('\r');
-                                break;
-                            // tab
-                            case 't':
-                                add('\t');
-                                break;
-
-                            // unicode escapes
-                            case 'u':
-                            {
-                                int codepoint;
-                                int codepoint1 = get_codepoint();
-
-                                if (JSON_UNLIKELY(codepoint1 == -1))
-                                {
-                                    error_message = "invalid string: '\\u' must be followed by 4 hex digits";
-                                    return token_type::parse_error;
-                                }
-
-                                // check if code point is a high surrogate
-                                if (0xD800 <= codepoint1 and codepoint1 <= 0xDBFF)
-                                {
-                                    // expect next \uxxxx entry
-                                    if (JSON_LIKELY(get() == '\\' and get() == 'u'))
-                                    {
-                                        const int codepoint2 = get_codepoint();
-
-                                        if (JSON_UNLIKELY(codepoint2 == -1))
-                                        {
-                                            error_message = "invalid string: '\\u' must be followed by 4 hex digits";
-                                            return token_type::parse_error;
-                                        }
-
-                                        // check if codepoint2 is a low surrogate
-                                        if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF))
-                                        {
-                                            codepoint =
-                                                // high surrogate occupies the most significant 22 bits
-                                                (codepoint1 << 10)
-                                                // low surrogate occupies the least significant 15 bits
-                                                + codepoint2
-                                                // there is still the 0xD800, 0xDC00 and 0x10000 noise
-                                                // in the result so we have to subtract with:
-                                                // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00
-                                                - 0x35FDC00;
-                                        }
-                                        else
-                                        {
-                                            error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
-                                            return token_type::parse_error;
-                                        }
-                                    }
-                                    else
-                                    {
-                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF";
-                                        return token_type::parse_error;
-                                    }
-                                }
-                                else
-                                {
-                                    if (JSON_UNLIKELY(0xDC00 <= codepoint1 and codepoint1 <= 0xDFFF))
-                                    {
-                                        error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF";
-                                        return token_type::parse_error;
-                                    }
-
-                                    // only work with first code point
-                                    codepoint = codepoint1;
-                                }
-
-                                // result of the above calculation yields a proper codepoint
-                                assert(0x00 <= codepoint and codepoint <= 0x10FFFF);
-
-                                // translate code point to bytes
-                                if (codepoint < 0x80)
-                                {
-                                    // 1-byte characters: 0xxxxxxx (ASCII)
-                                    add(codepoint);
-                                }
-                                else if (codepoint <= 0x7ff)
-                                {
-                                    // 2-byte characters: 110xxxxx 10xxxxxx
-                                    add(0xC0 | (codepoint >> 6));
-                                    add(0x80 | (codepoint & 0x3F));
-                                }
-                                else if (codepoint <= 0xffff)
-                                {
-                                    // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx
-                                    add(0xE0 | (codepoint >> 12));
-                                    add(0x80 | ((codepoint >> 6) & 0x3F));
-                                    add(0x80 | (codepoint & 0x3F));
-                                }
-                                else
-                                {
-                                    // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-                                    add(0xF0 | (codepoint >> 18));
-                                    add(0x80 | ((codepoint >> 12) & 0x3F));
-                                    add(0x80 | ((codepoint >> 6) & 0x3F));
-                                    add(0x80 | (codepoint & 0x3F));
-                                }
-
-                                break;
-                            }
-
-                            // other characters after escape
-                            default:
-                                error_message = "invalid string: forbidden character after backslash";
-                                return token_type::parse_error;
-                        }
-
-                        break;
-                    }
-
-                    // invalid control characters
-                    case 0x00:
-                    case 0x01:
-                    case 0x02:
-                    case 0x03:
-                    case 0x04:
-                    case 0x05:
-                    case 0x06:
-                    case 0x07:
-                    case 0x08:
-                    case 0x09:
-                    case 0x0a:
-                    case 0x0b:
-                    case 0x0c:
-                    case 0x0d:
-                    case 0x0e:
-                    case 0x0f:
-                    case 0x10:
-                    case 0x11:
-                    case 0x12:
-                    case 0x13:
-                    case 0x14:
-                    case 0x15:
-                    case 0x16:
-                    case 0x17:
-                    case 0x18:
-                    case 0x19:
-                    case 0x1a:
-                    case 0x1b:
-                    case 0x1c:
-                    case 0x1d:
-                    case 0x1e:
-                    case 0x1f:
-                    {
-                        error_message = "invalid string: control character must be escaped";
-                        return token_type::parse_error;
-                    }
-
-                    // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace))
-                    case 0x20:
-                    case 0x21:
-                    case 0x23:
-                    case 0x24:
-                    case 0x25:
-                    case 0x26:
-                    case 0x27:
-                    case 0x28:
-                    case 0x29:
-                    case 0x2a:
-                    case 0x2b:
-                    case 0x2c:
-                    case 0x2d:
-                    case 0x2e:
-                    case 0x2f:
-                    case 0x30:
-                    case 0x31:
-                    case 0x32:
-                    case 0x33:
-                    case 0x34:
-                    case 0x35:
-                    case 0x36:
-                    case 0x37:
-                    case 0x38:
-                    case 0x39:
-                    case 0x3a:
-                    case 0x3b:
-                    case 0x3c:
-                    case 0x3d:
-                    case 0x3e:
-                    case 0x3f:
-                    case 0x40:
-                    case 0x41:
-                    case 0x42:
-                    case 0x43:
-                    case 0x44:
-                    case 0x45:
-                    case 0x46:
-                    case 0x47:
-                    case 0x48:
-                    case 0x49:
-                    case 0x4a:
-                    case 0x4b:
-                    case 0x4c:
-                    case 0x4d:
-                    case 0x4e:
-                    case 0x4f:
-                    case 0x50:
-                    case 0x51:
-                    case 0x52:
-                    case 0x53:
-                    case 0x54:
-                    case 0x55:
-                    case 0x56:
-                    case 0x57:
-                    case 0x58:
-                    case 0x59:
-                    case 0x5a:
-                    case 0x5b:
-                    case 0x5d:
-                    case 0x5e:
-                    case 0x5f:
-                    case 0x60:
-                    case 0x61:
-                    case 0x62:
-                    case 0x63:
-                    case 0x64:
-                    case 0x65:
-                    case 0x66:
-                    case 0x67:
-                    case 0x68:
-                    case 0x69:
-                    case 0x6a:
-                    case 0x6b:
-                    case 0x6c:
-                    case 0x6d:
-                    case 0x6e:
-                    case 0x6f:
-                    case 0x70:
-                    case 0x71:
-                    case 0x72:
-                    case 0x73:
-                    case 0x74:
-                    case 0x75:
-                    case 0x76:
-                    case 0x77:
-                    case 0x78:
-                    case 0x79:
-                    case 0x7a:
-                    case 0x7b:
-                    case 0x7c:
-                    case 0x7d:
-                    case 0x7e:
-                    case 0x7f:
-                    {
-                        add(current);
-                        break;
-                    }
-
-                    // U+0080..U+07FF: bytes C2..DF 80..BF
-                    case 0xc2:
-                    case 0xc3:
-                    case 0xc4:
-                    case 0xc5:
-                    case 0xc6:
-                    case 0xc7:
-                    case 0xc8:
-                    case 0xc9:
-                    case 0xca:
-                    case 0xcb:
-                    case 0xcc:
-                    case 0xcd:
-                    case 0xce:
-                    case 0xcf:
-                    case 0xd0:
-                    case 0xd1:
-                    case 0xd2:
-                    case 0xd3:
-                    case 0xd4:
-                    case 0xd5:
-                    case 0xd6:
-                    case 0xd7:
-                    case 0xd8:
-                    case 0xd9:
-                    case 0xda:
-                    case 0xdb:
-                    case 0xdc:
-                    case 0xdd:
-                    case 0xde:
-                    case 0xdf:
-                    {
-                        add(current);
-                        get();
-                        if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                        {
-                            add(current);
-                            continue;
-                        }
-
-                        error_message = "invalid string: ill-formed UTF-8 byte";
-                        return token_type::parse_error;
-                    }
-
-                    // U+0800..U+0FFF: bytes E0 A0..BF 80..BF
-                    case 0xe0:
-                    {
-                        add(current);
-                        get();
-                        if (JSON_LIKELY(0xa0 <= current and current <= 0xbf))
-                        {
-                            add(current);
-                            get();
-                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                            {
-                                add(current);
-                                continue;
-                            }
-                        }
-
-                        error_message = "invalid string: ill-formed UTF-8 byte";
-                        return token_type::parse_error;
-                    }
-
-                    // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF
-                    // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF
-                    case 0xe1:
-                    case 0xe2:
-                    case 0xe3:
-                    case 0xe4:
-                    case 0xe5:
-                    case 0xe6:
-                    case 0xe7:
-                    case 0xe8:
-                    case 0xe9:
-                    case 0xea:
-                    case 0xeb:
-                    case 0xec:
-                    case 0xee:
-                    case 0xef:
-                    {
-                        add(current);
-                        get();
-                        if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                        {
-                            add(current);
-                            get();
-                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                            {
-                                add(current);
-                                continue;
-                            }
-                        }
-
-                        error_message = "invalid string: ill-formed UTF-8 byte";
-                        return token_type::parse_error;
-                    }
-
-                    // U+D000..U+D7FF: bytes ED 80..9F 80..BF
-                    case 0xed:
-                    {
-                        add(current);
-                        get();
-                        if (JSON_LIKELY(0x80 <= current and current <= 0x9f))
-                        {
-                            add(current);
-                            get();
-                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                            {
-                                add(current);
-                                continue;
-                            }
-                        }
-
-                        error_message = "invalid string: ill-formed UTF-8 byte";
-                        return token_type::parse_error;
-                    }
-
-                    // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
-                    case 0xf0:
-                    {
-                        add(current);
-                        get();
-                        if (JSON_LIKELY(0x90 <= current and current <= 0xbf))
-                        {
-                            add(current);
-                            get();
-                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                            {
-                                add(current);
-                                get();
-                                if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                                {
-                                    add(current);
-                                    continue;
-                                }
-                            }
-                        }
-
-                        error_message = "invalid string: ill-formed UTF-8 byte";
-                        return token_type::parse_error;
-                    }
-
-                    // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
-                    case 0xf1:
-                    case 0xf2:
-                    case 0xf3:
-                    {
-                        add(current);
-                        get();
-                        if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                        {
-                            add(current);
-                            get();
-                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                            {
-                                add(current);
-                                get();
-                                if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                                {
-                                    add(current);
-                                    continue;
-                                }
-                            }
-                        }
-
-                        error_message = "invalid string: ill-formed UTF-8 byte";
-                        return token_type::parse_error;
-                    }
-
-                    // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
-                    case 0xf4:
-                    {
-                        add(current);
-                        get();
-                        if (JSON_LIKELY(0x80 <= current and current <= 0x8f))
-                        {
-                            add(current);
-                            get();
-                            if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                            {
-                                add(current);
-                                get();
-                                if (JSON_LIKELY(0x80 <= current and current <= 0xbf))
-                                {
-                                    add(current);
-                                    continue;
-                                }
-                            }
-                        }
-
-                        error_message = "invalid string: ill-formed UTF-8 byte";
-                        return token_type::parse_error;
-                    }
-
-                    // remaining bytes (80..C1 and F5..FF) are ill-formed
-                    default:
-                    {
-                        error_message = "invalid string: ill-formed UTF-8 byte";
-                        return token_type::parse_error;
-                    }
-                }
-            }
-        }
-
-        static void strtof(float& f, const char* str, char** endptr) noexcept
-        {
-            f = std::strtof(str, endptr);
-        }
-
-        static void strtof(double& f, const char* str, char** endptr) noexcept
-        {
-            f = std::strtod(str, endptr);
-        }
-
-        static void strtof(long double& f, const char* str, char** endptr) noexcept
-        {
-            f = std::strtold(str, endptr);
-        }
-
-        /*!
-        @brief scan a number literal
-
-        This function scans a string according to Sect. 6 of RFC 7159.
-
-        The function is realized with a deterministic finite state machine
-        derived from the grammar described in RFC 7159. Starting in state
-        "init", the input is read and used to determined the next state. Only
-        state "done" accepts the number. State "error" is a trap state to model
-        errors. In the table below, "anything" means any character but the ones
-        listed before.
-
-        state    | 0        | 1-9      | e E      | +       | -       | .        | anything
-        ---------|----------|----------|----------|---------|---------|----------|-----------
-        init     | zero     | any1     | [error]  | [error] | minus   | [error]  | [error]
-        minus    | zero     | any1     | [error]  | [error] | [error] | [error]  | [error]
-        zero     | done     | done     | exponent | done    | done    | decimal1 | done
-        any1     | any1     | any1     | exponent | done    | done    | decimal1 | done
-        decimal1 | decimal2 | [error]  | [error]  | [error] | [error] | [error]  | [error]
-        decimal2 | decimal2 | decimal2 | exponent | done    | done    | done     | done
-        exponent | any2     | any2     | [error]  | sign    | sign    | [error]  | [error]
-        sign     | any2     | any2     | [error]  | [error] | [error] | [error]  | [error]
-        any2     | any2     | any2     | done     | done    | done    | done     | done
-
-        The state machine is realized with one label per state (prefixed with
-        "scan_number_") and `goto` statements between them. The state machine
-        contains cycles, but any cycle can be left when EOF is read. Therefore,
-        the function is guaranteed to terminate.
-
-        During scanning, the read bytes are stored in yytext. This string is
-        then converted to a signed integer, an unsigned integer, or a
-        floating-point number.
-
-        @return token_type::value_unsigned, token_type::value_integer, or
-                token_type::value_float if number could be successfully scanned,
-                token_type::parse_error otherwise
-
-        @note The scanner is independent of the current locale. Internally, the
-              locale's decimal point is used instead of `.` to work with the
-              locale-dependent converters.
-        */
-        token_type scan_number()
-        {
-            // reset yytext to store the number's bytes
-            reset();
-
-            // the type of the parsed number; initially set to unsigned; will be
-            // changed if minus sign, decimal point or exponent is read
-            token_type number_type = token_type::value_unsigned;
-
-            // state (init): we just found out we need to scan a number
-            switch (current)
-            {
-                case '-':
-                {
-                    add(current);
-                    goto scan_number_minus;
-                }
-
-                case '0':
-                {
-                    add(current);
-                    goto scan_number_zero;
-                }
-
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                {
-                    add(current);
-                    goto scan_number_any1;
-                }
-
-                default:
-                {
-                    // all other characters are rejected outside scan_number()
-                    assert(false);  // LCOV_EXCL_LINE
-                }
-            }
-
-scan_number_minus:
-            // state: we just parsed a leading minus sign
-            number_type = token_type::value_integer;
-            switch (get())
-            {
-                case '0':
-                {
-                    add(current);
-                    goto scan_number_zero;
-                }
-
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                {
-                    add(current);
-                    goto scan_number_any1;
-                }
-
-                default:
-                {
-                    error_message = "invalid number; expected digit after '-'";
-                    return token_type::parse_error;
-                }
-            }
-
-scan_number_zero:
-            // state: we just parse a zero (maybe with a leading minus sign)
-            switch (get())
-            {
-                case '.':
-                {
-                    add(decimal_point_char);
-                    goto scan_number_decimal1;
-                }
-
-                case 'e':
-                case 'E':
-                {
-                    add(current);
-                    goto scan_number_exponent;
-                }
-
-                default:
-                {
-                    goto scan_number_done;
-                }
-            }
-
-scan_number_any1:
-            // state: we just parsed a number 0-9 (maybe with a leading minus sign)
-            switch (get())
-            {
-                case '0':
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                {
-                    add(current);
-                    goto scan_number_any1;
-                }
-
-                case '.':
-                {
-                    add(decimal_point_char);
-                    goto scan_number_decimal1;
-                }
-
-                case 'e':
-                case 'E':
-                {
-                    add(current);
-                    goto scan_number_exponent;
-                }
-
-                default:
-                {
-                    goto scan_number_done;
-                }
-            }
-
-scan_number_decimal1:
-            // state: we just parsed a decimal point
-            number_type = token_type::value_float;
-            switch (get())
-            {
-                case '0':
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                {
-                    add(current);
-                    goto scan_number_decimal2;
-                }
-
-                default:
-                {
-                    error_message = "invalid number; expected digit after '.'";
-                    return token_type::parse_error;
-                }
-            }
-
-scan_number_decimal2:
-            // we just parsed at least one number after a decimal point
-            switch (get())
-            {
-                case '0':
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                {
-                    add(current);
-                    goto scan_number_decimal2;
-                }
-
-                case 'e':
-                case 'E':
-                {
-                    add(current);
-                    goto scan_number_exponent;
-                }
-
-                default:
-                {
-                    goto scan_number_done;
-                }
-            }
-
-scan_number_exponent:
-            // we just parsed an exponent
-            number_type = token_type::value_float;
-            switch (get())
-            {
-                case '+':
-                case '-':
-                {
-                    add(current);
-                    goto scan_number_sign;
-                }
-
-                case '0':
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                {
-                    add(current);
-                    goto scan_number_any2;
-                }
-
-                default:
-                {
-                    error_message = "invalid number; expected '+', '-', or digit after exponent";
-                    return token_type::parse_error;
-                }
-            }
-
-scan_number_sign:
-            // we just parsed an exponent sign
-            switch (get())
-            {
-                case '0':
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                {
-                    add(current);
-                    goto scan_number_any2;
-                }
-
-                default:
-                {
-                    error_message = "invalid number; expected digit after exponent sign";
-                    return token_type::parse_error;
-                }
-            }
-
-scan_number_any2:
-            // we just parsed a number after the exponent or exponent sign
-            switch (get())
-            {
-                case '0':
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                {
-                    add(current);
-                    goto scan_number_any2;
-                }
-
-                default:
-                {
-                    goto scan_number_done;
-                }
-            }
-
-scan_number_done:
-            // unget the character after the number (we only read it to know
-            // that we are done scanning a number)
-            --chars_read;
-            next_unget = true;
-
-            // terminate token
-            add('\0');
-            --yylen;
-
-            // try to parse integers first and fall back to floats
-            if (number_type == token_type::value_unsigned)
-            {
-                char* endptr = nullptr;
-                errno = 0;
-                const auto x = std::strtoull(yytext.data(), &endptr, 10);
-
-                // we checked the number format before
-                assert(endptr == yytext.data() + yylen);
-
-                if (errno == 0)
-                {
-                    value_unsigned = static_cast<number_unsigned_t>(x);
-                    if (value_unsigned == x)
-                    {
-                        return token_type::value_unsigned;
-                    }
-                }
-            }
-            else if (number_type == token_type::value_integer)
-            {
-                char* endptr = nullptr;
-                errno = 0;
-                const auto x = std::strtoll(yytext.data(), &endptr, 10);
-
-                // we checked the number format before
-                assert(endptr == yytext.data() + yylen);
-
-                if (errno == 0)
-                {
-                    value_integer = static_cast<number_integer_t>(x);
-                    if (value_integer == x)
-                    {
-                        return token_type::value_integer;
-                    }
-                }
-            }
-
-            // this code is reached if we parse a floating-point number or if
-            // an integer conversion above failed
-            strtof(value_float, yytext.data(), nullptr);
-            return token_type::value_float;
-        }
-
-        /*!
-        @param[in] literal_text  the literal text to expect
-        @param[in] length        the length of the passed literal text
-        @param[in] return_type   the token type to return on success
-        */
-        token_type scan_literal(const char* literal_text, const size_t length,
-                                token_type return_type)
-        {
-            assert(current == literal_text[0]);
-            for (size_t i = 1; i < length; ++i)
-            {
-                if (JSON_UNLIKELY(get() != literal_text[i]))
-                {
-                    error_message = "invalid literal";
-                    return token_type::parse_error;
-                }
-            }
-            return return_type;
-        }
-
-        /////////////////////
-        // input management
-        /////////////////////
-
-        /// reset yytext
-        void reset() noexcept
-        {
-            yylen = 0;
-            start_pos = chars_read - 1;
-        }
-
-        /// get a character from the input
-        int get()
-        {
-            ++chars_read;
-            return next_unget
-                   ? (next_unget = false, current)
-                   : (current = ia->get_character());
-        }
-
-        /// add a character to yytext
-        void add(int c)
-        {
-            // resize yytext if necessary; this condition is deemed unlikely,
-            // because we start with a 1024-byte buffer
-            if (JSON_UNLIKELY((yylen + 1 > yytext.capacity())))
-            {
-                yytext.resize(2 * yytext.capacity(), '\0');
-            }
-            assert(yylen < yytext.size());
-            yytext[yylen++] = static_cast<char>(c);
-        }
-
-      public:
-        /////////////////////
-        // value getters
-        /////////////////////
-
-        /// return integer value
-        constexpr number_integer_t get_number_integer() const noexcept
-        {
-            return value_integer;
-        }
-
-        /// return unsigned integer value
-        constexpr number_unsigned_t get_number_unsigned() const noexcept
-        {
-            return value_unsigned;
-        }
-
-        /// return floating-point value
-        constexpr number_float_t get_number_float() const noexcept
-        {
-            return value_float;
-        }
-
-        /// return string value
-        const std::string get_string()
-        {
-            // yytext cannot be returned as char*, because it may contain a
-            // null byte (parsed as "\u0000")
-            return std::string(yytext.data(), yylen);
-        }
-
-        /////////////////////
-        // diagnostics
-        /////////////////////
-
-        /// return position of last read token
-        constexpr size_t get_position() const noexcept
-        {
-            return chars_read;
-        }
-
-        /// return the last read token (for errors only)
-        std::string get_token_string() const
-        {
-            // get the raw byte sequence of the last token
-            std::string s = ia->read(start_pos, chars_read - start_pos);
-
-            // escape control characters
-            std::string result;
-            for (auto c : s)
-            {
-                if (c == '\0' or c == std::char_traits<char>::eof())
-                {
-                    // ignore EOF
-                    continue;
-                }
-                else if ('\x00' <= c and c <= '\x1f')
-                {
-                    // escape control characters
-                    std::stringstream ss;
-                    ss << "<U+" << std::setw(4) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(c) << ">";
-                    result += ss.str();
-                }
-                else
-                {
-                    // add character as is
-                    result.append(1, c);
-                }
-            }
-
-            return result;
-        }
-
-        /// return syntax error message
-        constexpr const char* get_error_message() const noexcept
-        {
-            return error_message;
-        }
-
-        /////////////////////
-        // actual scanner
-        /////////////////////
-
-        token_type scan()
-        {
-            // read next character and ignore whitespace
-            do
-            {
-                get();
-            }
-            while (current == ' ' or current == '\t' or current == '\n' or current == '\r');
-
-            switch (current)
-            {
-                // structural characters
-                case '[':
-                    return token_type::begin_array;
-                case ']':
-                    return token_type::end_array;
-                case '{':
-                    return token_type::begin_object;
-                case '}':
-                    return token_type::end_object;
-                case ':':
-                    return token_type::name_separator;
-                case ',':
-                    return token_type::value_separator;
-
-                // literals
-                case 't':
-                    return scan_literal("true", 4, token_type::literal_true);
-                case 'f':
-                    return scan_literal("false", 5, token_type::literal_false);
-                case 'n':
-                    return scan_literal("null", 4, token_type::literal_null);
-
-                // string
-                case '\"':
-                    return scan_string();
-
-                // number
-                case '-':
-                case '0':
-                case '1':
-                case '2':
-                case '3':
-                case '4':
-                case '5':
-                case '6':
-                case '7':
-                case '8':
-                case '9':
-                    return scan_number();
-
-                // end of input (the null byte is needed when parsing from
-                // string literals)
-                case '\0':
-                case std::char_traits<char>::eof():
-                    return token_type::end_of_input;
-
-                // error
-                default:
-                    error_message = "invalid literal";
-                    return token_type::parse_error;
-            }
-        }
-
-      private:
-        /// input adapter
-        input_adapter_t ia = nullptr;
-
-        /// the current character
-        int current = std::char_traits<char>::eof();
-
-        /// whether get() should return the last character again
-        bool next_unget = false;
-
-        /// the number of characters read
-        size_t chars_read = 0;
-        /// the start position of the current token
-        size_t start_pos = 0;
-
-        /// buffer for variable-length tokens (numbers, strings)
-        std::vector<char> yytext = std::vector<char>(1024, '\0');
-        /// current index in yytext
-        size_t yylen = 0;
-
-        /// a description of occurred lexer errors
-        const char* error_message = "";
-
-        // number values
-        number_integer_t value_integer = 0;
-        number_unsigned_t value_unsigned = 0;
-        number_float_t value_float = 0;
-
-        /// the decimal point
-        const char decimal_point_char = '.';
-    };
-
-    /*!
-    @brief syntax analysis
-
-    This class implements a recursive decent parser.
-    */
-    class parser
-    {
-      public:
-        /// a parser reading from an input adapter
-        explicit parser(input_adapter_t adapter,
-                        const parser_callback_t cb = nullptr)
-            : callback(cb), m_lexer(adapter)
-        {}
-
-        /*!
-        @brief public parser interface
-
-        @param[in] strict  whether to expect the last token to be EOF
-        @return parsed JSON value
-
-        @throw parse_error.101 in case of an unexpected token
-        @throw parse_error.102 if to_unicode fails or surrogate error
-        @throw parse_error.103 if to_unicode fails
-        */
-        basic_json parse(const bool strict = true)
-        {
-            // read first token
-            get_token();
-
-            basic_json result = parse_internal(true);
-            result.assert_invariant();
-
-            if (strict)
-            {
-                get_token();
-                expect(lexer::token_type::end_of_input);
-            }
-
-            // return parser result and replace it with null in case the
-            // top-level value was discarded by the callback function
-            return result.is_discarded() ? basic_json() : std::move(result);
-        }
-
-        /*!
-        @brief public accept interface
-
-        @param[in] strict  whether to expect the last token to be EOF
-        @return whether the input is a proper JSON text
-        */
-        bool accept(const bool strict = true)
-        {
-            // read first token
-            get_token();
-
-            if (not accept_internal())
-            {
-                return false;
-            }
-
-            if (strict and get_token() != lexer::token_type::end_of_input)
-            {
-                return false;
-            }
-
-            return true;
-        }
-
-      private:
-        /*!
-        @brief the actual parser
-        @throw parse_error.101 in case of an unexpected token
-        @throw parse_error.102 if to_unicode fails or surrogate error
-        @throw parse_error.103 if to_unicode fails
-        */
-        basic_json parse_internal(bool keep)
-        {
-            auto result = basic_json(value_t::discarded);
-
-            switch (last_token)
-            {
-                case lexer::token_type::begin_object:
-                {
-                    if (keep and (not callback
-                                  or ((keep = callback(depth++, parse_event_t::object_start, result)) != 0)))
-                    {
-                        // explicitly set result to object to cope with {}
-                        result.m_type = value_t::object;
-                        result.m_value = value_t::object;
-                    }
-
-                    // read next token
-                    get_token();
-
-                    // closing } -> we are done
-                    if (last_token == lexer::token_type::end_object)
-                    {
-                        if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
-                        {
-                            result = basic_json(value_t::discarded);
-                        }
-                        return result;
-                    }
-
-                    // parse values
-                    while (true)
-                    {
-                        // store key
-                        expect(lexer::token_type::value_string);
-                        const auto key = m_lexer.get_string();
-
-                        bool keep_tag = false;
-                        if (keep)
-                        {
-                            if (callback)
-                            {
-                                basic_json k(key);
-                                keep_tag = callback(depth, parse_event_t::key, k);
-                            }
-                            else
-                            {
-                                keep_tag = true;
-                            }
-                        }
-
-                        // parse separator (:)
-                        get_token();
-                        expect(lexer::token_type::name_separator);
-
-                        // parse and add value
-                        get_token();
-                        auto value = parse_internal(keep);
-                        if (keep and keep_tag and not value.is_discarded())
-                        {
-                            result[key] = std::move(value);
-                        }
-
-                        // comma -> next value
-                        get_token();
-                        if (last_token == lexer::token_type::value_separator)
-                        {
-                            get_token();
-                            continue;
-                        }
-
-                        // closing }
-                        expect(lexer::token_type::end_object);
-                        break;
-                    }
-
-                    if (keep and callback and not callback(--depth, parse_event_t::object_end, result))
-                    {
-                        result = basic_json(value_t::discarded);
-                    }
-
-                    return result;
-                }
-
-                case lexer::token_type::begin_array:
-                {
-                    if (keep and (not callback
-                                  or ((keep = callback(depth++, parse_event_t::array_start, result)) != 0)))
-                    {
-                        // explicitly set result to object to cope with []
-                        result.m_type = value_t::array;
-                        result.m_value = value_t::array;
-                    }
-
-                    // read next token
-                    get_token();
-
-                    // closing ] -> we are done
-                    if (last_token == lexer::token_type::end_array)
-                    {
-                        if (callback and not callback(--depth, parse_event_t::array_end, result))
-                        {
-                            result = basic_json(value_t::discarded);
-                        }
-                        return result;
-                    }
-
-                    // parse values
-                    while (true)
-                    {
-                        // parse value
-                        auto value = parse_internal(keep);
-                        if (keep and not value.is_discarded())
-                        {
-                            result.push_back(std::move(value));
-                        }
-
-                        // comma -> next value
-                        get_token();
-                        if (last_token == lexer::token_type::value_separator)
-                        {
-                            get_token();
-                            continue;
-                        }
-
-                        // closing ]
-                        expect(lexer::token_type::end_array);
-                        break;
-                    }
-
-                    if (keep and callback and not callback(--depth, parse_event_t::array_end, result))
-                    {
-                        result = basic_json(value_t::discarded);
-                    }
-
-                    return result;
-                }
-
-                case lexer::token_type::literal_null:
-                {
-                    result.m_type = value_t::null;
-                    break;
-                }
-
-                case lexer::token_type::value_string:
-                {
-                    result = basic_json(m_lexer.get_string());
-                    break;
-                }
-
-                case lexer::token_type::literal_true:
-                {
-                    result.m_type = value_t::boolean;
-                    result.m_value = true;
-                    break;
-                }
-
-                case lexer::token_type::literal_false:
-                {
-                    result.m_type = value_t::boolean;
-                    result.m_value = false;
-                    break;
-                }
-
-                case lexer::token_type::value_unsigned:
-                {
-                    result.m_type = value_t::number_unsigned;
-                    result.m_value = m_lexer.get_number_unsigned();
-                    break;
-                }
-
-                case lexer::token_type::value_integer:
-                {
-                    result.m_type = value_t::number_integer;
-                    result.m_value = m_lexer.get_number_integer();
-                    break;
-                }
-
-                case lexer::token_type::value_float:
-                {
-                    result.m_type = value_t::number_float;
-                    result.m_value = m_lexer.get_number_float();
-
-                    // throw in case of infinity or NAN
-                    if (JSON_UNLIKELY(not std::isfinite(result.m_value.number_float)))
-                    {
-                        JSON_THROW(out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'"));
-                    }
-
-                    break;
-                }
-
-                case lexer::token_type::parse_error:
-                {
-                    // using "uninitialized" to avoid "expected" message
-                    expect(lexer::token_type::uninitialized);
-                }
-
-                default:
-                {
-                    // the last token was unexpected; we expected a value
-                    expect(lexer::token_type::literal_or_value);
-                }
-            }
-
-            if (keep and callback and not callback(depth, parse_event_t::value, result))
-            {
-                result = basic_json(value_t::discarded);
-            }
-            return result;
-        }
-
-        /*!
-        @brief the acutal acceptor
-
-        @invariant 1. The last token is not yet processed. Therefore, the
-                      caller of this function must make sure a token has
-                      been read.
-                   2. When this function returns, the last token is processed.
-                      That is, the last read character was already considered.
-
-        This invariant makes sure that no token needs to be "unput".
-        */
-        bool accept_internal()
-        {
-            switch (last_token)
-            {
-                case lexer::token_type::begin_object:
-                {
-                    // read next token
-                    get_token();
-
-                    // closing } -> we are done
-                    if (last_token == lexer::token_type::end_object)
-                    {
-                        return true;
-                    }
-
-                    // parse values
-                    while (true)
-                    {
-                        // parse key
-                        if (last_token != lexer::token_type::value_string)
-                        {
-                            return false;
-                        }
-
-                        // parse separator (:)
-                        get_token();
-                        if (last_token != lexer::token_type::name_separator)
-                        {
-                            return false;
-                        }
-
-                        // parse value
-                        get_token();
-                        if (not accept_internal())
-                        {
-                            return false;
-                        }
-
-                        // comma -> next value
-                        get_token();
-                        if (last_token == lexer::token_type::value_separator)
-                        {
-                            get_token();
-                            continue;
-                        }
-
-                        // closing }
-                        if (last_token != lexer::token_type::end_object)
-                        {
-                            return false;
-                        }
-
-                        return true;
-                    }
-                }
-
-                case lexer::token_type::begin_array:
-                {
-                    // read next token
-                    get_token();
-
-                    // closing ] -> we are done
-                    if (last_token == lexer::token_type::end_array)
-                    {
-                        return true;
-                    }
-
-                    // parse values
-                    while (true)
-                    {
-                        // parse value
-                        if (not accept_internal())
-                        {
-                            return false;
-                        }
-
-                        // comma -> next value
-                        get_token();
-                        if (last_token == lexer::token_type::value_separator)
-                        {
-                            get_token();
-                            continue;
-                        }
-
-                        // closing ]
-                        if (last_token != lexer::token_type::end_array)
-                        {
-                            return false;
-                        }
-
-                        return true;
-                    }
-                }
-
-                case lexer::token_type::literal_false:
-                case lexer::token_type::literal_null:
-                case lexer::token_type::literal_true:
-                case lexer::token_type::value_float:
-                case lexer::token_type::value_integer:
-                case lexer::token_type::value_string:
-                case lexer::token_type::value_unsigned:
-                {
-                    return true;
-                }
-
-                default:
-                {
-                    // the last token was unexpected
-                    return false;
-                }
-            }
-        }
-
-        /// get next token from lexer
-        typename lexer::token_type get_token()
-        {
-            return (last_token = m_lexer.scan());
-        }
-
-        /*!
-        @throw parse_error.101 if expected token did not occur
-        */
-        void expect(typename lexer::token_type t)
-        {
-            if (JSON_UNLIKELY(t != last_token))
-            {
-                errored = true;
-                expected = t;
-                throw_exception();
-            }
-        }
-
-        [[noreturn]] void throw_exception() const
-        {
-            std::string error_msg = "syntax error - ";
-            if (last_token == lexer::token_type::parse_error)
-            {
-                error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" + m_lexer.get_token_string() + "'";
-            }
-            else
-            {
-                error_msg += "unexpected " + std::string(lexer::token_type_name(last_token));
-            }
-
-            if (expected != lexer::token_type::uninitialized)
-            {
-                error_msg += "; expected " + std::string(lexer::token_type_name(expected));
-            }
-
-            JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg));
-        }
-
-      private:
-        /// current level of recursion
-        int depth = 0;
-        /// callback function
-        const parser_callback_t callback = nullptr;
-        /// the type of the last read token
-        typename lexer::token_type last_token = lexer::token_type::uninitialized;
-        /// the lexer
-        lexer m_lexer;
-        /// whether a syntax error occurred
-        bool errored = false;
-        /// possible reason for the syntax error
-        typename lexer::token_type expected = lexer::token_type::uninitialized;
-    };
-
-  public:
-    /*!
-    @brief JSON Pointer
-
-    A JSON pointer defines a string syntax for identifying a specific value
-    within a JSON document. It can be used with functions `at` and
-    `operator[]`. Furthermore, JSON pointers are the base for JSON patches.
-
-    @sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
-
-    @since version 2.0.0
-    */
-    class json_pointer
-    {
-        /// allow basic_json to access private members
-        friend class basic_json;
-
-      public:
-        /*!
-        @brief create JSON pointer
-
-        Create a JSON pointer according to the syntax described in
-        [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3).
-
-        @param[in] s  string representing the JSON pointer; if omitted, the
-                      empty string is assumed which references the whole JSON
-                      value
-
-        @throw parse_error.107 if the given JSON pointer @a s is nonempty and
-        does not begin with a slash (`/`); see example below
-
-        @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s
-        is not followed by `0` (representing `~`) or `1` (representing `/`);
-        see example below
-
-        @liveexample{The example shows the construction several valid JSON
-        pointers as well as the exceptional behavior.,json_pointer}
-
-        @since version 2.0.0
-        */
-        explicit json_pointer(const std::string& s = "")
-            : reference_tokens(split(s))
-        {}
-
-        /*!
-        @brief return a string representation of the JSON pointer
-
-        @invariant For each JSON pointer `ptr`, it holds:
-        @code {.cpp}
-        ptr == json_pointer(ptr.to_string());
-        @endcode
-
-        @return a string representation of the JSON pointer
-
-        @liveexample{The example shows the result of `to_string`.,
-        json_pointer__to_string}
-
-        @since version 2.0.0
-        */
-        std::string to_string() const noexcept
-        {
-            return std::accumulate(reference_tokens.begin(),
-                                   reference_tokens.end(), std::string{},
-                                   [](const std::string & a, const std::string & b)
-            {
-                return a + "/" + escape(b);
-            });
-        }
-
-        /// @copydoc to_string()
-        operator std::string() const
-        {
-            return to_string();
-        }
-
-      private:
-        /*!
-        @brief remove and return last reference pointer
-        @throw out_of_range.405 if JSON pointer has no parent
-        */
-        std::string pop_back()
-        {
-            if (is_root())
-            {
-                JSON_THROW(out_of_range::create(405, "JSON pointer has no parent"));
-            }
-
-            auto last = reference_tokens.back();
-            reference_tokens.pop_back();
-            return last;
-        }
-
-        /// return whether pointer points to the root document
-        bool is_root() const
-        {
-            return reference_tokens.empty();
-        }
-
-        json_pointer top() const
-        {
-            if (is_root())
-            {
-                JSON_THROW(out_of_range::create(405, "JSON pointer has no parent"));
-            }
-
-            json_pointer result = *this;
-            result.reference_tokens = {reference_tokens[0]};
-            return result;
-        }
-
-        /*!
-        @brief create and return a reference to the pointed to value
-
-        @complexity Linear in the number of reference tokens.
-
-        @throw parse_error.109 if array index is not a number
-        @throw type_error.313 if value cannot be unflattened
-        */
-        reference get_and_create(reference j) const
-        {
-            pointer result = &j;
-
-            // in case no reference tokens exist, return a reference to the
-            // JSON value j which will be overwritten by a primitive value
-            for (const auto& reference_token : reference_tokens)
-            {
-                switch (result->m_type)
-                {
-                    case value_t::null:
-                    {
-                        if (reference_token == "0")
-                        {
-                            // start a new array if reference token is 0
-                            result = &result->operator[](0);
-                        }
-                        else
-                        {
-                            // start a new object otherwise
-                            result = &result->operator[](reference_token);
-                        }
-                        break;
-                    }
-
-                    case value_t::object:
-                    {
-                        // create an entry in the object
-                        result = &result->operator[](reference_token);
-                        break;
-                    }
-
-                    case value_t::array:
-                    {
-                        // create an entry in the array
-                        JSON_TRY
-                        {
-                            result = &result->operator[](static_cast<size_type>(std::stoi(reference_token)));
-                        }
-                        JSON_CATCH (std::invalid_argument&)
-                        {
-                            JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                        }
-                        break;
-                    }
-
-                    /*
-                    The following code is only reached if there exists a
-                    reference token _and_ the current value is primitive. In
-                    this case, we have an error situation, because primitive
-                    values may only occur as single value; that is, with an
-                    empty list of reference tokens.
-                    */
-                    default:
-                    {
-                        JSON_THROW(type_error::create(313, "invalid value to unflatten"));
-                    }
-                }
-            }
-
-            return *result;
-        }
-
-        /*!
-        @brief return a reference to the pointed to value
-
-        @note This version does not throw if a value is not present, but tries
-        to create nested values instead. For instance, calling this function
-        with pointer `"/this/that"` on a null value is equivalent to calling
-        `operator[]("this").operator[]("that")` on that value, effectively
-        changing the null value to an object.
-
-        @param[in] ptr  a JSON value
-
-        @return reference to the JSON value pointed to by the JSON pointer
-
-        @complexity Linear in the length of the JSON pointer.
-
-        @throw parse_error.106   if an array index begins with '0'
-        @throw parse_error.109   if an array index was not a number
-        @throw out_of_range.404  if the JSON pointer can not be resolved
-        */
-        reference get_unchecked(pointer ptr) const
-        {
-            for (const auto& reference_token : reference_tokens)
-            {
-                // convert null values to arrays or objects before continuing
-                if (ptr->m_type == value_t::null)
-                {
-                    // check if reference token is a number
-                    const bool nums = std::all_of(reference_token.begin(),
-                                                  reference_token.end(),
-                                                  [](const char x)
-                    {
-                        return (x >= '0' and x <= '9');
-                    });
-
-                    // change value to array for numbers or "-" or to object
-                    // otherwise
-                    if (nums or reference_token == "-")
-                    {
-                        *ptr = value_t::array;
-                    }
-                    else
-                    {
-                        *ptr = value_t::object;
-                    }
-                }
-
-                switch (ptr->m_type)
-                {
-                    case value_t::object:
-                    {
-                        // use unchecked object access
-                        ptr = &ptr->operator[](reference_token);
-                        break;
-                    }
-
-                    case value_t::array:
-                    {
-                        // error condition (cf. RFC 6901, Sect. 4)
-                        if (reference_token.size() > 1 and reference_token[0] == '0')
-                        {
-                            JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'"));
-                        }
-
-                        if (reference_token == "-")
-                        {
-                            // explicitly treat "-" as index beyond the end
-                            ptr = &ptr->operator[](ptr->m_value.array->size());
-                        }
-                        else
-                        {
-                            // convert array index to number; unchecked access
-                            JSON_TRY
-                            {
-                                ptr = &ptr->operator[](static_cast<size_type>(std::stoi(reference_token)));
-                            }
-                            JSON_CATCH (std::invalid_argument&)
-                            {
-                                JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                            }
-                        }
-                        break;
-                    }
-
-                    default:
-                    {
-                        JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-                    }
-                }
-            }
-
-            return *ptr;
-        }
-
-        /*!
-        @throw parse_error.106   if an array index begins with '0'
-        @throw parse_error.109   if an array index was not a number
-        @throw out_of_range.402  if the array index '-' is used
-        @throw out_of_range.404  if the JSON pointer can not be resolved
-        */
-        reference get_checked(pointer ptr) const
-        {
-            for (const auto& reference_token : reference_tokens)
-            {
-                switch (ptr->m_type)
-                {
-                    case value_t::object:
-                    {
-                        // note: at performs range check
-                        ptr = &ptr->at(reference_token);
-                        break;
-                    }
-
-                    case value_t::array:
-                    {
-                        if (reference_token == "-")
-                        {
-                            // "-" always fails the range check
-                            JSON_THROW(out_of_range::create(402, "array index '-' (" +
-                                                            std::to_string(ptr->m_value.array->size()) +
-                                                            ") is out of range"));
-                        }
-
-                        // error condition (cf. RFC 6901, Sect. 4)
-                        if (reference_token.size() > 1 and reference_token[0] == '0')
-                        {
-                            JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'"));
-                        }
-
-                        // note: at performs range check
-                        JSON_TRY
-                        {
-                            ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
-                        }
-                        JSON_CATCH (std::invalid_argument&)
-                        {
-                            JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                        }
-                        break;
-                    }
-
-                    default:
-                    {
-                        JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-                    }
-                }
-            }
-
-            return *ptr;
-        }
-
-        /*!
-        @brief return a const reference to the pointed to value
-
-        @param[in] ptr  a JSON value
-
-        @return const reference to the JSON value pointed to by the JSON
-                pointer
-
-        @throw parse_error.106   if an array index begins with '0'
-        @throw parse_error.109   if an array index was not a number
-        @throw out_of_range.402  if the array index '-' is used
-        @throw out_of_range.404  if the JSON pointer can not be resolved
-        */
-        const_reference get_unchecked(const_pointer ptr) const
-        {
-            for (const auto& reference_token : reference_tokens)
-            {
-                switch (ptr->m_type)
-                {
-                    case value_t::object:
-                    {
-                        // use unchecked object access
-                        ptr = &ptr->operator[](reference_token);
-                        break;
-                    }
-
-                    case value_t::array:
-                    {
-                        if (reference_token == "-")
-                        {
-                            // "-" cannot be used for const access
-                            JSON_THROW(out_of_range::create(402, "array index '-' (" +
-                                                            std::to_string(ptr->m_value.array->size()) +
-                                                            ") is out of range"));
-                        }
-
-                        // error condition (cf. RFC 6901, Sect. 4)
-                        if (reference_token.size() > 1 and reference_token[0] == '0')
-                        {
-                            JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'"));
-                        }
-
-                        // use unchecked array access
-                        JSON_TRY
-                        {
-                            ptr = &ptr->operator[](static_cast<size_type>(std::stoi(reference_token)));
-                        }
-                        JSON_CATCH (std::invalid_argument&)
-                        {
-                            JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                        }
-                        break;
-                    }
-
-                    default:
-                    {
-                        JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-                    }
-                }
-            }
-
-            return *ptr;
-        }
-
-        /*!
-        @throw parse_error.106   if an array index begins with '0'
-        @throw parse_error.109   if an array index was not a number
-        @throw out_of_range.402  if the array index '-' is used
-        @throw out_of_range.404  if the JSON pointer can not be resolved
-        */
-        const_reference get_checked(const_pointer ptr) const
-        {
-            for (const auto& reference_token : reference_tokens)
-            {
-                switch (ptr->m_type)
-                {
-                    case value_t::object:
-                    {
-                        // note: at performs range check
-                        ptr = &ptr->at(reference_token);
-                        break;
-                    }
-
-                    case value_t::array:
-                    {
-                        if (reference_token == "-")
-                        {
-                            // "-" always fails the range check
-                            JSON_THROW(out_of_range::create(402, "array index '-' (" +
-                                                            std::to_string(ptr->m_value.array->size()) +
-                                                            ") is out of range"));
-                        }
-
-                        // error condition (cf. RFC 6901, Sect. 4)
-                        if (reference_token.size() > 1 and reference_token[0] == '0')
-                        {
-                            JSON_THROW(parse_error::create(106, 0, "array index '" + reference_token + "' must not begin with '0'"));
-                        }
-
-                        // note: at performs range check
-                        JSON_TRY
-                        {
-                            ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
-                        }
-                        JSON_CATCH (std::invalid_argument&)
-                        {
-                            JSON_THROW(parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
-                        }
-                        break;
-                    }
-
-                    default:
-                    {
-                        JSON_THROW(out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
-                    }
-                }
-            }
-
-            return *ptr;
-        }
-
-        /*!
-        @brief split the string input to reference tokens
-
-        @note This function is only called by the json_pointer constructor.
-              All exceptions below are documented there.
-
-        @throw parse_error.107  if the pointer is not empty or begins with '/'
-        @throw parse_error.108  if character '~' is not followed by '0' or '1'
-        */
-        static std::vector<std::string> split(const std::string& reference_string)
-        {
-            std::vector<std::string> result;
-
-            // special case: empty reference string -> no reference tokens
-            if (reference_string.empty())
-            {
-                return result;
-            }
-
-            // check if nonempty reference string begins with slash
-            if (reference_string[0] != '/')
-            {
-                JSON_THROW(parse_error::create(107, 1, "JSON pointer must be empty or begin with '/' - was: '" + reference_string + "'"));
-            }
-
-            // extract the reference tokens:
-            // - slash: position of the last read slash (or end of string)
-            // - start: position after the previous slash
-            for (
-                // search for the first slash after the first character
-                size_t slash = reference_string.find_first_of('/', 1),
-                // set the beginning of the first reference token
-                start = 1;
-                // we can stop if start == string::npos+1 = 0
-                start != 0;
-                // set the beginning of the next reference token
-                // (will eventually be 0 if slash == std::string::npos)
-                start = slash + 1,
-                // find next slash
-                slash = reference_string.find_first_of('/', start))
-            {
-                // use the text between the beginning of the reference token
-                // (start) and the last slash (slash).
-                auto reference_token = reference_string.substr(start, slash - start);
-
-                // check reference tokens are properly escaped
-                for (size_t pos = reference_token.find_first_of('~');
-                        pos != std::string::npos;
-                        pos = reference_token.find_first_of('~', pos + 1))
-                {
-                    assert(reference_token[pos] == '~');
-
-                    // ~ must be followed by 0 or 1
-                    if (pos == reference_token.size() - 1 or
-                            (reference_token[pos + 1] != '0' and
-                             reference_token[pos + 1] != '1'))
-                    {
-                        JSON_THROW(parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'"));
-                    }
-                }
-
-                // finally, store the reference token
-                unescape(reference_token);
-                result.push_back(reference_token);
-            }
-
-            return result;
-        }
-
-        /*!
-        @brief replace all occurrences of a substring by another string
-
-        @param[in,out] s  the string to manipulate; changed so that all
-                          occurrences of @a f are replaced with @a t
-        @param[in]     f  the substring to replace with @a t
-        @param[in]     t  the string to replace @a f
-
-        @pre The search string @a f must not be empty. **This precondition is
-             enforced with an assertion.**
-
-        @since version 2.0.0
-        */
-        static void replace_substring(std::string& s,
-                                      const std::string& f,
-                                      const std::string& t)
-        {
-            assert(not f.empty());
-
-            for (
-                size_t pos = s.find(f);         // find first occurrence of f
-                pos != std::string::npos;       // make sure f was found
-                s.replace(pos, f.size(), t),    // replace with t
-                pos = s.find(f, pos + t.size()) // find next occurrence of f
-            );
-        }
-
-        /// escape tilde and slash
-        static std::string escape(std::string s)
-        {
-            // escape "~"" to "~0" and "/" to "~1"
-            replace_substring(s, "~", "~0");
-            replace_substring(s, "/", "~1");
-            return s;
-        }
-
-        /// unescape tilde and slash
-        static void unescape(std::string& s)
-        {
-            // first transform any occurrence of the sequence '~1' to '/'
-            replace_substring(s, "~1", "/");
-            // then transform any occurrence of the sequence '~0' to '~'
-            replace_substring(s, "~0", "~");
-        }
-
-        /*!
-        @param[in] reference_string  the reference string to the current value
-        @param[in] value             the value to consider
-        @param[in,out] result        the result object to insert values to
-
-        @note Empty objects or arrays are flattened to `null`.
-        */
-        static void flatten(const std::string& reference_string,
-                            const basic_json& value,
-                            basic_json& result)
-        {
-            switch (value.m_type)
-            {
-                case value_t::array:
-                {
-                    if (value.m_value.array->empty())
-                    {
-                        // flatten empty array as null
-                        result[reference_string] = nullptr;
-                    }
-                    else
-                    {
-                        // iterate array and use index as reference string
-                        for (size_t i = 0; i < value.m_value.array->size(); ++i)
-                        {
-                            flatten(reference_string + "/" + std::to_string(i),
-                                    value.m_value.array->operator[](i), result);
-                        }
-                    }
-                    break;
-                }
-
-                case value_t::object:
-                {
-                    if (value.m_value.object->empty())
-                    {
-                        // flatten empty object as null
-                        result[reference_string] = nullptr;
-                    }
-                    else
-                    {
-                        // iterate object and use keys as reference string
-                        for (const auto& element : *value.m_value.object)
-                        {
-                            flatten(reference_string + "/" + escape(element.first),
-                                    element.second, result);
-                        }
-                    }
-                    break;
-                }
-
-                default:
-                {
-                    // add primitive value with its reference string
-                    result[reference_string] = value;
-                    break;
-                }
-            }
-        }
-
-        /*!
-        @param[in] value  flattened JSON
-
-        @return unflattened JSON
-
-        @throw parse_error.109 if array index is not a number
-        @throw type_error.314  if value is not an object
-        @throw type_error.315  if object values are not primitive
-        @throw type_error.313  if value cannot be unflattened
-        */
-        static basic_json unflatten(const basic_json& value)
-        {
-            if (not value.is_object())
-            {
-                JSON_THROW(type_error::create(314, "only objects can be unflattened"));
-            }
-
-            basic_json result;
-
-            // iterate the JSON object values
-            for (const auto& element : *value.m_value.object)
-            {
-                if (not element.second.is_primitive())
-                {
-                    JSON_THROW(type_error::create(315, "values in object must be primitive"));
-                }
-
-                // assign value to reference pointed to by JSON pointer; Note
-                // that if the JSON pointer is "" (i.e., points to the whole
-                // value), function get_and_create returns a reference to
-                // result itself. An assignment will then create a primitive
-                // value.
-                json_pointer(element.first).get_and_create(result) = element.second;
-            }
-
-            return result;
-        }
-
-        friend bool operator==(json_pointer const& lhs,
-                               json_pointer const& rhs) noexcept
-        {
-            return lhs.reference_tokens == rhs.reference_tokens;
-        }
-
-        friend bool operator!=(json_pointer const& lhs,
-                               json_pointer const& rhs) noexcept
-        {
-            return !(lhs == rhs);
-        }
-
-        /// the reference tokens
-        std::vector<std::string> reference_tokens {};
-    };
-
     //////////////////////////
     // JSON Pointer support //
     //////////////////////////
@@ -13961,7 +13735,7 @@ scan_number_done:
     @complexity Linear in the size the JSON value.
 
     @throw type_error.314  if value is not an object
-    @throw type_error.315  if object values are not primitve
+    @throw type_error.315  if object values are not primitive
 
     @liveexample{The following code shows how a flattened JSON object is
     unflattened into the original nested JSON object.,unflatten}
@@ -14110,7 +13884,7 @@ scan_number_done:
                         else
                         {
                             const auto idx = std::stoi(last_path);
-                            if (static_cast<size_type>(idx) > parent.size())
+                            if (JSON_UNLIKELY(static_cast<size_type>(idx) > parent.size()))
                             {
                                 // avoid undefined behavior
                                 JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
@@ -14145,7 +13919,7 @@ scan_number_done:
             {
                 // perform range check
                 auto it = parent.find(last_path);
-                if (it != parent.end())
+                if (JSON_LIKELY(it != parent.end()))
                 {
                     parent.erase(it);
                 }
@@ -14162,7 +13936,7 @@ scan_number_done:
         };
 
         // type check: top level value must be an array
-        if (not json_patch.is_array())
+        if (JSON_UNLIKELY(not json_patch.is_array()))
         {
             JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
         }
@@ -14182,13 +13956,13 @@ scan_number_done:
                 const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'";
 
                 // check if desired value is present
-                if (it == val.m_value.object->end())
+                if (JSON_UNLIKELY(it == val.m_value.object->end()))
                 {
                     JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'"));
                 }
 
                 // check if result is of type string
-                if (string_type and not it->second.is_string())
+                if (JSON_UNLIKELY(string_type and not it->second.is_string()))
                 {
                     JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'"));
                 }
@@ -14198,7 +13972,7 @@ scan_number_done:
             };
 
             // type check: every element of the array must be an object
-            if (not val.is_object())
+            if (JSON_UNLIKELY(not val.is_object()))
             {
                 JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects"));
             }
@@ -14271,7 +14045,7 @@ scan_number_done:
                     }
 
                     // throw an exception if test fails
-                    if (not success)
+                    if (JSON_UNLIKELY(not success))
                     {
                         JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump()));
                     }
@@ -14323,8 +14097,7 @@ scan_number_done:
 
     @since version 2.0.0
     */
-    static basic_json diff(const basic_json& source,
-                           const basic_json& target,
+    static basic_json diff(const basic_json& source, const basic_json& target,
                            const std::string& path = "")
     {
         // the patch
@@ -14341,9 +14114,7 @@ scan_number_done:
             // different types: replace value
             result.push_back(
             {
-                {"op", "replace"},
-                {"path", path},
-                {"value", target}
+                {"op", "replace"}, {"path", path}, {"value", target}
             });
         }
         else
@@ -14353,7 +14124,7 @@ scan_number_done:
                 case value_t::array:
                 {
                     // first pass: traverse common elements
-                    size_t i = 0;
+                    std::size_t i = 0;
                     while (i < source.size() and i < target.size())
                     {
                         // recursive call to compare array values at index i
@@ -14413,8 +14184,7 @@ scan_number_done:
                             // found a key that is not in o -> remove it
                             result.push_back(object(
                             {
-                                {"op", "remove"},
-                                {"path", path + "/" + key}
+                                {"op", "remove"}, {"path", path + "/" + key}
                             }));
                         }
                     }
@@ -14428,8 +14198,7 @@ scan_number_done:
                             const auto key = json_pointer::escape(it.key());
                             result.push_back(
                             {
-                                {"op", "add"},
-                                {"path", path + "/" + key},
+                                {"op", "add"}, {"path", path + "/" + key},
                                 {"value", it.value()}
                             });
                         }
@@ -14443,9 +14212,7 @@ scan_number_done:
                     // both primitive type: replace value
                     result.push_back(
                     {
-                        {"op", "replace"},
-                        {"path", path},
-                        {"value", target}
+                        {"op", "replace"}, {"path", path}, {"value", target}
                     });
                     break;
                 }
@@ -14471,6 +14238,400 @@ uses the standard template types.
 @since version 1.0.0
 */
 using json = basic_json<>;
+
+//////////////////
+// json_pointer //
+//////////////////
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    auto result = &j;
+
+    // in case no reference tokens exist, return a reference to the JSON value
+    // j which will be overwritten by a primitive value
+    for (const auto& reference_token : reference_tokens)
+    {
+        switch (result->m_type)
+        {
+            case detail::value_t::null:
+            {
+                if (reference_token == "0")
+                {
+                    // start a new array if reference token is 0
+                    result = &result->operator[](0);
+                }
+                else
+                {
+                    // start a new object otherwise
+                    result = &result->operator[](reference_token);
+                }
+                break;
+            }
+
+            case detail::value_t::object:
+            {
+                // create an entry in the object
+                result = &result->operator[](reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                // create an entry in the array
+                JSON_TRY
+                {
+                    result = &result->operator[](static_cast<size_type>(std::stoi(reference_token)));
+                }
+                JSON_CATCH(std::invalid_argument&)
+                {
+                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                }
+                break;
+            }
+
+            /*
+            The following code is only reached if there exists a reference
+            token _and_ the current value is primitive. In this case, we have
+            an error situation, because primitive values may only occur as
+            single value; that is, with an empty list of reference tokens.
+            */
+            default:
+                JSON_THROW(detail::type_error::create(313, "invalid value to unflatten"));
+        }
+    }
+
+    return *result;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    for (const auto& reference_token : reference_tokens)
+    {
+        // convert null values to arrays or objects before continuing
+        if (ptr->m_type == detail::value_t::null)
+        {
+            // check if reference token is a number
+            const bool nums =
+                std::all_of(reference_token.begin(), reference_token.end(),
+                            [](const char x)
+            {
+                return (x >= '0' and x <= '9');
+            });
+
+            // change value to array for numbers or "-" or to object otherwise
+            *ptr = (nums or reference_token == "-")
+                   ? detail::value_t::array
+                   : detail::value_t::object;
+        }
+
+        switch (ptr->m_type)
+        {
+            case detail::value_t::object:
+            {
+                // use unchecked object access
+                ptr = &ptr->operator[](reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                // error condition (cf. RFC 6901, Sect. 4)
+                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                {
+                    JSON_THROW(detail::parse_error::create(106, 0,
+                                                           "array index '" + reference_token +
+                                                           "' must not begin with '0'"));
+                }
+
+                if (reference_token == "-")
+                {
+                    // explicitly treat "-" as index beyond the end
+                    ptr = &ptr->operator[](ptr->m_value.array->size());
+                }
+                else
+                {
+                    // convert array index to number; unchecked access
+                    JSON_TRY
+                    {
+                        ptr = &ptr->operator[](
+                            static_cast<size_type>(std::stoi(reference_token)));
+                    }
+                    JSON_CATCH(std::invalid_argument&)
+                    {
+                        JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                    }
+                }
+                break;
+            }
+
+            default:
+                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+        }
+    }
+
+    return *ptr;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    for (const auto& reference_token : reference_tokens)
+    {
+        switch (ptr->m_type)
+        {
+            case detail::value_t::object:
+            {
+                // note: at performs range check
+                ptr = &ptr->at(reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                if (JSON_UNLIKELY(reference_token == "-"))
+                {
+                    // "-" always fails the range check
+                    JSON_THROW(detail::out_of_range::create(402,
+                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                            ") is out of range"));
+                }
+
+                // error condition (cf. RFC 6901, Sect. 4)
+                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                {
+                    JSON_THROW(detail::parse_error::create(106, 0,
+                                                           "array index '" + reference_token +
+                                                           "' must not begin with '0'"));
+                }
+
+                // note: at performs range check
+                JSON_TRY
+                {
+                    ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                }
+                JSON_CATCH(std::invalid_argument&)
+                {
+                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                }
+                break;
+            }
+
+            default:
+                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+        }
+    }
+
+    return *ptr;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+const NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    for (const auto& reference_token : reference_tokens)
+    {
+        switch (ptr->m_type)
+        {
+            case detail::value_t::object:
+            {
+                // use unchecked object access
+                ptr = &ptr->operator[](reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                if (JSON_UNLIKELY(reference_token == "-"))
+                {
+                    // "-" cannot be used for const access
+                    JSON_THROW(detail::out_of_range::create(402,
+                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                            ") is out of range"));
+                }
+
+                // error condition (cf. RFC 6901, Sect. 4)
+                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                {
+                    JSON_THROW(detail::parse_error::create(106, 0,
+                                                           "array index '" + reference_token +
+                                                           "' must not begin with '0'"));
+                }
+
+                // use unchecked array access
+                JSON_TRY
+                {
+                    ptr = &ptr->operator[](
+                        static_cast<size_type>(std::stoi(reference_token)));
+                }
+                JSON_CATCH(std::invalid_argument&)
+                {
+                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                }
+                break;
+            }
+
+            default:
+                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+        }
+    }
+
+    return *ptr;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+const NLOHMANN_BASIC_JSON_TPL&
+json_pointer::get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const
+{
+    using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type;
+    for (const auto& reference_token : reference_tokens)
+    {
+        switch (ptr->m_type)
+        {
+            case detail::value_t::object:
+            {
+                // note: at performs range check
+                ptr = &ptr->at(reference_token);
+                break;
+            }
+
+            case detail::value_t::array:
+            {
+                if (JSON_UNLIKELY(reference_token == "-"))
+                {
+                    // "-" always fails the range check
+                    JSON_THROW(detail::out_of_range::create(402,
+                                                            "array index '-' (" + std::to_string(ptr->m_value.array->size()) +
+                                                            ") is out of range"));
+                }
+
+                // error condition (cf. RFC 6901, Sect. 4)
+                if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0'))
+                {
+                    JSON_THROW(detail::parse_error::create(106, 0,
+                                                           "array index '" + reference_token +
+                                                           "' must not begin with '0'"));
+                }
+
+                // note: at performs range check
+                JSON_TRY
+                {
+                    ptr = &ptr->at(static_cast<size_type>(std::stoi(reference_token)));
+                }
+                JSON_CATCH(std::invalid_argument&)
+                {
+                    JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number"));
+                }
+                break;
+            }
+
+            default:
+                JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'"));
+        }
+    }
+
+    return *ptr;
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+void json_pointer::flatten(const std::string& reference_string,
+                           const NLOHMANN_BASIC_JSON_TPL& value,
+                           NLOHMANN_BASIC_JSON_TPL& result)
+{
+    switch (value.m_type)
+    {
+        case detail::value_t::array:
+        {
+            if (value.m_value.array->empty())
+            {
+                // flatten empty array as null
+                result[reference_string] = nullptr;
+            }
+            else
+            {
+                // iterate array and use index as reference string
+                for (std::size_t i = 0; i < value.m_value.array->size(); ++i)
+                {
+                    flatten(reference_string + "/" + std::to_string(i),
+                            value.m_value.array->operator[](i), result);
+                }
+            }
+            break;
+        }
+
+        case detail::value_t::object:
+        {
+            if (value.m_value.object->empty())
+            {
+                // flatten empty object as null
+                result[reference_string] = nullptr;
+            }
+            else
+            {
+                // iterate object and use keys as reference string
+                for (const auto& element : *value.m_value.object)
+                {
+                    flatten(reference_string + "/" + escape(element.first), element.second, result);
+                }
+            }
+            break;
+        }
+
+        default:
+        {
+            // add primitive value with its reference string
+            result[reference_string] = value;
+            break;
+        }
+    }
+}
+
+NLOHMANN_BASIC_JSON_TPL_DECLARATION
+NLOHMANN_BASIC_JSON_TPL
+json_pointer::unflatten(const NLOHMANN_BASIC_JSON_TPL& value)
+{
+    if (JSON_UNLIKELY(not value.is_object()))
+    {
+        JSON_THROW(detail::type_error::create(314, "only objects can be unflattened"));
+    }
+
+    NLOHMANN_BASIC_JSON_TPL result;
+
+    // iterate the JSON object values
+    for (const auto& element : *value.m_value.object)
+    {
+        if (JSON_UNLIKELY(not element.second.is_primitive()))
+        {
+            JSON_THROW(detail::type_error::create(315, "values in object must be primitive"));
+        }
+
+        // assign value to reference pointed to by JSON pointer; Note that if
+        // the JSON pointer is "" (i.e., points to the whole value), function
+        // get_and_create returns a reference to result itself. An assignment
+        // will then create a primitive value.
+        json_pointer(element.first).get_and_create(result) = element.second;
+    }
+
+    return result;
+}
+
+inline bool operator==(json_pointer const& lhs, json_pointer const& rhs) noexcept
+{
+    return (lhs.reference_tokens == rhs.reference_tokens);
+}
+
+inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcept
+{
+    return not (lhs == rhs);
+}
 } // namespace nlohmann
 
 
@@ -14514,8 +14675,10 @@ struct hash<nlohmann::json>
 };
 
 /// specialization for std::less<value_t>
-template <>
-struct less<::nlohmann::detail::value_t>
+/// @note: do not remove the space after '<',
+///        see https://github.com/nlohmann/json/pull/679
+template<>
+struct less< ::nlohmann::detail::value_t>
 {
     /*!
     @brief compare two value_t enum values
@@ -14581,5 +14744,7 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std
 #undef JSON_LIKELY
 #undef JSON_UNLIKELY
 #undef JSON_DEPRECATED
+#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION
+#undef NLOHMANN_BASIC_JSON_TPL
 
 #endif
diff --git a/lib/serialisation/JSON_IO.cc b/lib/serialisation/JSON_IO.cc
index 23a78b3e..99a9cdd6 100644
--- a/lib/serialisation/JSON_IO.cc
+++ b/lib/serialisation/JSON_IO.cc
@@ -76,10 +76,9 @@ void JSONWriter::delete_comma()
 // annoying, but necessary for TravisCI
 namespace Grid
 {
-  template<>
   void JSONWriter::writeDefault(const std::string &s,	const std::string &x)
   {
-  //std::cout << "JSONWriter::writeDefault(string) : " << s <<  std::endl;
+    //std::cout << "JSONWriter::writeDefault(string) : " << s <<  std::endl;
     std::ostringstream os;
     os << std::boolalpha << x;
     if (s.size())
diff --git a/lib/serialisation/JSON_IO.h b/lib/serialisation/JSON_IO.h
index 23b9a836..00287447 100644
--- a/lib/serialisation/JSON_IO.h
+++ b/lib/serialisation/JSON_IO.h
@@ -64,6 +64,8 @@ namespace Grid
     template<std::size_t N>
     void writeDefault(const std::string &s, const char(&x)[N]);
 
+    void writeDefault(const std::string &s, const std::string &x);
+
 
   private:
     void delete_comma();
@@ -120,22 +122,6 @@ namespace Grid
      ss_ << os.str() << " ," ;
   }
 
-
-  // specialize for string
-  template <>
-  void JSONWriter::writeDefault(const std::string &s, const std::string &x)
-  {
-    //std::cout << "JSONWriter::writeDefault(string) : " << s <<  std::endl;
-    std::ostringstream os;
-    os << std::boolalpha << x;
-    if (s.size())
-      ss_ << "\""<< s << "\" : \"" << os.str() << "\" ," ;
-    else
-     ss_ << os.str() << " ," ;
-  }
-
-
-
   template <typename U>
   void JSONWriter::writeDefault(const std::string &s, const std::complex<U> &x)
   {

From 5918769f9719491d357c29f4ac18bf5be0b1e3d1 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sat, 16 Sep 2017 12:51:26 +0100
Subject: [PATCH 094/115] Subtle Naik term bug updated in Stencil; less on
 logical && with a function call on right

---
 lib/stencil/Stencil.h                  | 16 +++--
 tests/solver/Test_staggered_cg_prec.cc | 87 ++++++++++++++++++++++++++
 2 files changed, 97 insertions(+), 6 deletions(-)
 create mode 100644 tests/solver/Test_staggered_cg_prec.cc

diff --git a/lib/stencil/Stencil.h b/lib/stencil/Stencil.h
index cd0792d5..887d8a7c 100644
--- a/lib/stencil/Stencil.h
+++ b/lib/stencil/Stencil.h
@@ -400,11 +400,13 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
       if ( sshift[0] == sshift[1] ) {
 	if (splice_dim) {
 	  splicetime-=usecond();
-	  same_node = same_node && GatherSimd(source,dimension,shift,0x3,compress,face_idx);
+	  auto tmp  = GatherSimd(source,dimension,shift,0x3,compress,face_idx);
+	  same_node = same_node && tmp;
 	  splicetime+=usecond();
 	} else { 
 	  nosplicetime-=usecond();
-	  same_node = same_node && Gather(source,dimension,shift,0x3,compress,face_idx);
+	  auto tmp  = Gather(source,dimension,shift,0x3,compress,face_idx);
+	  same_node = same_node && tmp;
 	  nosplicetime+=usecond();
 	}
       } else {
@@ -412,13 +414,15 @@ class CartesianStencil { // Stencil runs along coordinate axes only; NO diagonal
 	  splicetime-=usecond();
 	  // if checkerboard is unfavourable take two passes
 	  // both with block stride loop iteration
-	  same_node = same_node && GatherSimd(source,dimension,shift,0x1,compress,face_idx);
-	  same_node = same_node && GatherSimd(source,dimension,shift,0x2,compress,face_idx);
+	  auto tmp1 =  GatherSimd(source,dimension,shift,0x1,compress,face_idx);
+	  auto tmp2 =  GatherSimd(source,dimension,shift,0x2,compress,face_idx);
+	  same_node = same_node && tmp1 && tmp2;
 	  splicetime+=usecond();
 	} else {
 	  nosplicetime-=usecond();
-	  same_node = same_node && Gather(source,dimension,shift,0x1,compress,face_idx);
-	  same_node = same_node && Gather(source,dimension,shift,0x2,compress,face_idx);
+	  auto tmp1 = Gather(source,dimension,shift,0x1,compress,face_idx);
+	  auto tmp2 = Gather(source,dimension,shift,0x2,compress,face_idx);
+	  same_node = same_node && tmp1 && tmp2;
 	  nosplicetime+=usecond();
 	}
       }
diff --git a/tests/solver/Test_staggered_cg_prec.cc b/tests/solver/Test_staggered_cg_prec.cc
new file mode 100644
index 00000000..66f11d3d
--- /dev/null
+++ b/tests/solver/Test_staggered_cg_prec.cc
@@ -0,0 +1,87 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_wilson_cg_unprec.cc
+
+    Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+#include <Grid/algorithms/iterative/BlockConjugateGradient.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+template<class d>
+struct scal {
+  d internal;
+};
+
+  Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT
+  };
+
+int main (int argc, char ** argv)
+{
+  typedef typename ImprovedStaggeredFermionR::FermionField FermionField; 
+  typedef typename ImprovedStaggeredFermionR::ComplexField ComplexField; 
+  typename ImprovedStaggeredFermionR::ImplParams params; 
+
+  Grid_init(&argc,&argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+  GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+
+  std::vector<int> seeds({1,2,3,4});
+  GridParallelRNG          pRNG(&Grid);  pRNG.SeedFixedIntegers(seeds);
+
+  FermionField src(&Grid); random(pRNG,src);
+  RealD nrm = norm2(src);
+  LatticeGaugeField Umu(&Grid); SU3::HotConfiguration(pRNG,Umu);
+
+  double volume=1;
+  for(int mu=0;mu<Nd;mu++){
+    volume=volume*latt_size[mu];
+  }  
+  
+  RealD mass=0.1;
+  ImprovedStaggeredFermionR Ds(Umu,Umu,Grid,RBGrid,mass);
+
+  FermionField res_o(&RBGrid); 
+  FermionField src_o(&RBGrid); 
+  pickCheckerboard(Odd,src_o,src);
+  res_o=zero;
+
+  SchurDiagMooeeOperator<ImprovedStaggeredFermionR,FermionField> HermOpEO(Ds);
+  ConjugateGradient<FermionField> CG(1.0e-8,10000);
+  CG(HermOpEO,src_o,res_o);
+
+  Grid_finalize();
+}

From 17c5b0f1526ee3da8cf15547e7b85fad8c7dbfb2 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Sat, 16 Sep 2017 18:18:07 +0100
Subject: [PATCH 095/115] Patching comparison point

---
 benchmarks/Benchmark_ITT.cc | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/benchmarks/Benchmark_ITT.cc b/benchmarks/Benchmark_ITT.cc
index 2017f6d7..666e4830 100644
--- a/benchmarks/Benchmark_ITT.cc
+++ b/benchmarks/Benchmark_ITT.cc
@@ -701,12 +701,14 @@ int main (int argc, char ** argv)
   if ( do_su3 ) {
     // empty for now
   }
-
+#if 1
   int sel=2;
   std::vector<int> L_list({8,12,16,24});
-
-  //int sel=1;
-  //  std::vector<int> L_list({8,12});
+#else
+  int sel=1;
+  std::vector<int> L_list({8,12});
+#endif
+  int selm1=sel-1;
   std::vector<double> robust_list;
 
   std::vector<double> wilson;
@@ -785,7 +787,8 @@ int main (int argc, char ** argv)
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
 
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;
-  std::cout<<GridLogMessage << " Comparison point     result: "  << dwf4[sel]/NN << " Mflop/s per node"<<std::endl;
+  std::cout<<GridLogMessage << " Comparison point     result: "  << 0.5*(dwf4[sel]+dwf4[selm1])/NN << " Mflop/s per node"<<std::endl;
+  std::cout<<GridLogMessage << " Comparison point is 0.5*("<<dwf4[sel]/NN<<"+"<<dwf4[selm1]/NN << ") "<<std::endl;
   std::cout<<std::setprecision(3);
   std::cout<<GridLogMessage << " Comparison point robustness: "  << robust_list[sel] <<std::endl;
   std::cout<<GridLogMessage << "=================================================================================="<<std::endl;

From a9ec5cf564aafd737a9fbf224ae6ebf43b2ef780 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 21 Sep 2017 10:32:41 +0100
Subject: [PATCH 096/115] Christoph bug report integrate

---
 lib/tensors/Tensor_index.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/tensors/Tensor_index.h b/lib/tensors/Tensor_index.h
index f114baf8..500f6c8d 100644
--- a/lib/tensors/Tensor_index.h
+++ b/lib/tensors/Tensor_index.h
@@ -175,7 +175,7 @@ class TensorIndexRecursion {
       }
     }
   template<class vtype,int N> inline static 
-    void pokeIndex(iVector<vtype,N> &ret, const iVector<decltype(TensorIndexRecursion<Level-1>::peekIndex(ret._internal[0],0)),N> &arg, int i,int j)
+    void pokeIndex(iVector<vtype,N> &ret, const iVector<decltype(TensorIndexRecursion<Level-1>::peekIndex(ret._internal[0],0,0)),N> &arg, int i,int j)
     {
       for(int ii=0;ii<N;ii++){
 	TensorIndexRecursion<Level-1>::pokeIndex(ret._internal[ii],arg._internal[ii],i,j);
@@ -191,7 +191,7 @@ class TensorIndexRecursion {
       }}
     }
   template<class vtype,int N> inline static 
-    void pokeIndex(iMatrix<vtype,N> &ret, const iMatrix<decltype(TensorIndexRecursion<Level-1>::peekIndex(ret._internal[0][0],0)),N> &arg, int i,int j)
+    void pokeIndex(iMatrix<vtype,N> &ret, const iMatrix<decltype(TensorIndexRecursion<Level-1>::peekIndex(ret._internal[0][0],0,0)),N> &arg, int i,int j)
     {
       for(int ii=0;ii<N;ii++){
       for(int jj=0;jj<N;jj++){

From fddeb29d6b43048af8a0c40c6a8e4fb942fa0088 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Thu, 21 Sep 2017 11:10:08 +0100
Subject: [PATCH 097/115] Bug fix with spreadout FFT

---
 lib/algorithms/FFT.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/algorithms/FFT.h b/lib/algorithms/FFT.h
index 240f338b..ec558ad9 100644
--- a/lib/algorithms/FFT.h
+++ b/lib/algorithms/FFT.h
@@ -230,6 +230,7 @@ namespace Grid {
       // Barrel shift and collect global pencil
       std::vector<int> lcoor(Nd), gcoor(Nd);
       result = source;
+      int pc = processor_coor[dim];
       for(int p=0;p<processors[dim];p++) {
         PARALLEL_REGION
         {
@@ -240,7 +241,8 @@ namespace Grid {
           for(int idx=0;idx<sgrid->lSites();idx++) {
             sgrid->LocalIndexToLocalCoor(idx,cbuf);
             peekLocalSite(s,result,cbuf);
-            cbuf[dim]+=p*L;
+	    cbuf[dim]+=((pc+p) % processors[dim])*L;
+	    //            cbuf[dim]+=p*L;
             pokeLocalSite(s,pgbuf,cbuf);
           }
         }
@@ -278,7 +280,6 @@ namespace Grid {
       flops+= flops_call*NN;
       
       // writing out result
-      int pc = processor_coor[dim];
       PARALLEL_REGION
       {
         std::vector<int> clbuf(Nd), cgbuf(Nd);

From f7072d1ac2b5552998bf49b01a47128d67d7b682 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Mon, 2 Oct 2017 07:13:40 +0100
Subject: [PATCH 098/115] Solving an annoying compilation error in json

---
 lib/json/json.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/json/json.hpp b/lib/json/json.hpp
index e7c42920..9d589120 100644
--- a/lib/json/json.hpp
+++ b/lib/json/json.hpp
@@ -63,7 +63,7 @@ SOFTWARE.
         #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers"
     #endif
 #elif defined(__GNUC__)
-    #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900
+    #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40805
         #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers"
     #endif
 #endif

From 073525c5b3a32e6ab4567246893c08d8da609ac9 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 2 Oct 2017 03:38:21 -0700
Subject: [PATCH 099/115] Small patch from cori

---
 tests/solver/Test_dwf_mrhs_cg.cc | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc
index b4880148..2d2cfcb1 100644
--- a/tests/solver/Test_dwf_mrhs_cg.cc
+++ b/tests/solver/Test_dwf_mrhs_cg.cc
@@ -128,12 +128,11 @@ int main (int argc, char ** argv)
   {
     for(int n=0;n<nrhs;n++){
 
-      std::stringstream filefn;
-      filefn << filef << "."<< n;
-
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       std::cout << GridLogMessage << " Writing out record "<<n<<std::endl;
       std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+
+      std::stringstream filefn;      filefn << filef << "."<< n;
       ScidacWriter _ScidacWriter;
       _ScidacWriter.open(filefn.str());
       _ScidacWriter.writeScidacFieldRecord(src[n],record);
@@ -147,10 +146,8 @@ int main (int argc, char ** argv)
     std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
       
     for(int n=0;n<nrhs;n++){
-
-      std::stringstream filefn;
-      filefn << filef << "."<< n;
       if ( n==me ) { 
+	std::stringstream filefn;	filefn << filef << "."<< n;
 	ScidacReader  _ScidacReader;
 	_ScidacReader.open(filefn.str());
 	_ScidacReader.readScidacFieldRecord(s_src,record);

From e5c19e1fd728142fa7139bdf2135a9a1c9877075 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 2 Oct 2017 12:25:52 +0100
Subject: [PATCH 100/115] RB constructor change

---
 tests/solver/Test_staggered_cg_prec.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/solver/Test_staggered_cg_prec.cc b/tests/solver/Test_staggered_cg_prec.cc
index 66f11d3d..0e359c2d 100644
--- a/tests/solver/Test_staggered_cg_prec.cc
+++ b/tests/solver/Test_staggered_cg_prec.cc
@@ -57,7 +57,7 @@ int main (int argc, char ** argv)
   std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
   std::vector<int> mpi_layout  = GridDefaultMpi();
   GridCartesian               Grid(latt_size,simd_layout,mpi_layout);
-  GridRedBlackCartesian     RBGrid(latt_size,simd_layout,mpi_layout);
+  GridRedBlackCartesian     RBGrid(&Grid);
 
   std::vector<int> seeds({1,2,3,4});
   GridParallelRNG          pRNG(&Grid);  pRNG.SeedFixedIntegers(seeds);

From 1edcf902b73b66240b293f6fae94b991117da1b7 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 2 Oct 2017 12:41:02 +0100
Subject: [PATCH 101/115] Macos ANON

---
 lib/communicator/Communicator_base.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index c60f6c6d..836bc01e 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -147,8 +147,13 @@ void *CartesianCommunicator::ShmBufferTranslate(int rank,void * local_p) {
 }
 void CartesianCommunicator::ShmInitGeneric(void){
 #if 1
-
-  int mmap_flag = MAP_SHARED | MAP_ANONYMOUS;
+  int mmap_flag =0;
+#ifdef MAP_ANONYMOUS
+  mmap_flag = mmap_flag| MAP_SHARED | MAP_ANONYMOUS;
+#endif
+#ifdef MAP_ANON
+  mmap_flag = mmap_flag| MAP_SHARED | MAP_ANON;
+#endif
 #ifdef MAP_HUGETLB
   if ( Hugepages ) mmap_flag |= MAP_HUGETLB;
 #endif

From d54807b8c0cd1a7658ff8563bb00d1137b987e3e Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 2 Oct 2017 23:14:56 +0100
Subject: [PATCH 102/115] MPIT works with split grid now

---
 lib/communicator/Communicator_base.cc     | 99 +++++++++++++++++++++++
 lib/communicator/Communicator_base.h      |  3 +-
 lib/communicator/Communicator_mpi.cc      | 89 --------------------
 lib/communicator/Communicator_mpit.cc     | 27 -------
 lib/qcd/action/fermion/CayleyFermion5D.cc |  2 -
 5 files changed, 100 insertions(+), 120 deletions(-)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index 836bc01e..bcf429ab 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -96,6 +96,105 @@ void CartesianCommunicator::GlobalSumVector(ComplexD *c,int N)
   GlobalSumVector((double *)c,2*N);
 }
 
+
+#if defined( GRID_COMMS_MPI) || defined (GRID_COMMS_MPIT)
+
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,const CartesianCommunicator &parent) 
+{
+  _ndimension = processors.size();
+  assert(_ndimension = parent._ndimension);
+  
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  // split the communicator
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  int Nparent;
+  MPI_Comm_size(parent.communicator,&Nparent);
+
+  int childsize=1;
+  for(int d=0;d<processors.size();d++) {
+    childsize *= processors[d];
+  }
+  int Nchild = Nparent/childsize;
+  assert (childsize * Nchild == Nparent);
+
+  int prank;  MPI_Comm_rank(parent.communicator,&prank);
+  int crank = prank % childsize;
+  int ccomm = prank / childsize;
+
+  MPI_Comm comm_split;
+  if ( Nchild > 1 ) { 
+
+    std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec<<std::endl;
+    std::cout << GridLogMessage<<" parent grid["<< parent._ndimension<<"]    ";
+    for(int d=0;d<parent._processors.size();d++)  std::cout << parent._processors[d] << " ";
+    std::cout<<std::endl;
+
+    std::cout << GridLogMessage<<" child grid["<< _ndimension <<"]    ";
+    for(int d=0;d<processors.size();d++)  std::cout << processors[d] << " ";
+    std::cout<<std::endl;
+
+    int ierr= MPI_Comm_split(parent.communicator, ccomm,crank,&comm_split);
+    assert(ierr==0);
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    // Declare victory
+    //////////////////////////////////////////////////////////////////////////////////////////////////////
+    std::cout << GridLogMessage<<"Divided communicator "<< parent._Nprocessors<<" into "
+	      <<Nchild <<" communicators with " << childsize << " ranks"<<std::endl;
+  } else {
+    comm_split=parent.communicator;
+    //    std::cout << "Passed parental communicator to a new communicator" <<std::endl;
+  }
+
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Set up from the new split communicator
+  //////////////////////////////////////////////////////////////////////////////////////////////////////
+  InitFromMPICommunicator(processors,comm_split);
+}
+//////////////////////////////////////////////////////////////////////////////////////////////////////
+// Take an MPI_Comm and self assemble
+//////////////////////////////////////////////////////////////////////////////////////////////////////
+void CartesianCommunicator::InitFromMPICommunicator(const std::vector<int> &processors, MPI_Comm communicator_base)
+{
+  //  if ( communicator_base != communicator_world ) {
+  //    std::cout << "Cartesian communicator created with a non-world communicator"<<std::endl;
+  //  }
+  _ndimension = processors.size();
+  _processor_coor.resize(_ndimension);
+
+  /////////////////////////////////
+  // Count the requested nodes
+  /////////////////////////////////
+  _Nprocessors=1;
+  _processors = processors;
+  for(int i=0;i<_ndimension;i++){
+    _Nprocessors*=_processors[i];
+  }
+
+  std::vector<int> periodic(_ndimension,1);
+  MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],1,&communicator);
+  MPI_Comm_rank(communicator,&_processor);
+  MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
+
+  int Size;
+  MPI_Comm_size(communicator,&Size);
+
+#ifdef GRID_COMMS_MPIT
+  communicator_halo.resize (2*_ndimension);
+  for(int i=0;i<_ndimension*2;i++){
+    MPI_Comm_dup(communicator,&communicator_halo[i]);
+  }
+#endif
+  
+  assert(Size==_Nprocessors);
+}
+
+CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors) 
+{
+  InitFromMPICommunicator(processors,communicator_world);
+}
+
+#endif
+
 #if !defined( GRID_COMMS_MPI3) 
 
 int                      CartesianCommunicator::NodeCount(void)    { return ProcessorCount();};
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index e8791cd8..bfdb0da1 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -157,8 +157,7 @@ class CartesianCommunicator {
   CartesianCommunicator(const std::vector<int> &pdimensions_in);
 
  private:
-#if defined (GRID_COMMS_MPI) 
-  //|| defined (GRID_COMMS_MPI3) 
+#if defined (GRID_COMMS_MPI) || defined (GRID_COMMS_MPIT) 
   ////////////////////////////////////////////////
   // Private initialise from an MPI communicator
   // Can use after an MPI_Comm_split, but hidden from user so private
diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index 1a7a0c05..a55c0164 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -52,95 +52,6 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   MPI_Comm_dup (MPI_COMM_WORLD,&communicator_world);
   ShmInitGeneric();
 }
-
-CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors) 
-{
-  InitFromMPICommunicator(processors,communicator_world);
-  //  std::cout << "Passed communicator world to a new communicator" <<communicator<<std::endl;
-}
-CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,const CartesianCommunicator &parent) 
-{
-  _ndimension = processors.size();
-  assert(_ndimension = parent._ndimension);
-  
-  //////////////////////////////////////////////////////////////////////////////////////////////////////
-  // split the communicator
-  //////////////////////////////////////////////////////////////////////////////////////////////////////
-  int Nparent;
-  MPI_Comm_size(parent.communicator,&Nparent);
-
-  int childsize=1;
-  for(int d=0;d<processors.size();d++) {
-    childsize *= processors[d];
-  }
-  int Nchild = Nparent/childsize;
-  assert (childsize * Nchild == Nparent);
-
-  int prank;  MPI_Comm_rank(parent.communicator,&prank);
-  int crank = prank % childsize;
-  int ccomm = prank / childsize;
-
-  MPI_Comm comm_split;
-  if ( Nchild > 1 ) { 
-
-    std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec<<std::endl;
-    std::cout << GridLogMessage<<" parent grid["<< parent._ndimension<<"]    ";
-    for(int d=0;d<parent._processors.size();d++)  std::cout << parent._processors[d] << " ";
-    std::cout<<std::endl;
-
-    std::cout << GridLogMessage<<" child grid["<< _ndimension <<"]    ";
-    for(int d=0;d<processors.size();d++)  std::cout << processors[d] << " ";
-    std::cout<<std::endl;
-
-    int ierr= MPI_Comm_split(parent.communicator, ccomm,crank,&comm_split);
-    assert(ierr==0);
-    //////////////////////////////////////////////////////////////////////////////////////////////////////
-    // Declare victory
-    //////////////////////////////////////////////////////////////////////////////////////////////////////
-    std::cout << GridLogMessage<<"Divided communicator "<< parent._Nprocessors<<" into "
-	      <<Nchild <<" communicators with " << childsize << " ranks"<<std::endl;
-  } else {
-    comm_split=parent.communicator;
-    //    std::cout << "Passed parental communicator to a new communicator" <<std::endl;
-  }
-
-  //////////////////////////////////////////////////////////////////////////////////////////////////////
-  // Set up from the new split communicator
-  //////////////////////////////////////////////////////////////////////////////////////////////////////
-  InitFromMPICommunicator(processors,comm_split);
-
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////
-// Take an MPI_Comm and self assemble
-//////////////////////////////////////////////////////////////////////////////////////////////////////
-void CartesianCommunicator::InitFromMPICommunicator(const std::vector<int> &processors, MPI_Comm communicator_base)
-{
-  //  if ( communicator_base != communicator_world ) {
-  //    std::cout << "Cartesian communicator created with a non-world communicator"<<std::endl;
-  //  }
-  _ndimension = processors.size();
-  _processor_coor.resize(_ndimension);
-
-  /////////////////////////////////
-  // Count the requested nodes
-  /////////////////////////////////
-  _Nprocessors=1;
-  _processors = processors;
-  for(int i=0;i<_ndimension;i++){
-    _Nprocessors*=_processors[i];
-  }
-
-  std::vector<int> periodic(_ndimension,1);
-  MPI_Cart_create(communicator_base, _ndimension,&_processors[0],&periodic[0],1,&communicator);
-  MPI_Comm_rank(communicator,&_processor);
-  MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
-
-  int Size;
-  MPI_Comm_size(communicator,&Size);
-  
-  assert(Size==_Nprocessors);
-}
 void CartesianCommunicator::GlobalSum(uint32_t &u){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
   assert(ierr==0);
diff --git a/lib/communicator/Communicator_mpit.cc b/lib/communicator/Communicator_mpit.cc
index eb6ef87d..5137c27b 100644
--- a/lib/communicator/Communicator_mpit.cc
+++ b/lib/communicator/Communicator_mpit.cc
@@ -53,33 +53,6 @@ void CartesianCommunicator::Init(int *argc, char ***argv) {
   ShmInitGeneric();
 }
 
-CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors)
-{
-  _ndimension = processors.size();
-  std::vector<int> periodic(_ndimension,1);
-
-  _Nprocessors=1;
-  _processors = processors;
-  _processor_coor.resize(_ndimension);
-  
-  MPI_Cart_create(communicator_world, _ndimension,&_processors[0],&periodic[0],1,&communicator);
-  MPI_Comm_rank(communicator,&_processor);
-  MPI_Cart_coords(communicator,_processor,_ndimension,&_processor_coor[0]);
-
-  for(int i=0;i<_ndimension;i++){
-    _Nprocessors*=_processors[i];
-  }
-
-  communicator_halo.resize (2*_ndimension);
-  for(int i=0;i<_ndimension*2;i++){
-    MPI_Comm_dup(communicator,&communicator_halo[i]);
-  }
-  
-  int Size; 
-  MPI_Comm_size(communicator,&Size);
-  
-  assert(Size==_Nprocessors);
-}
 void CartesianCommunicator::GlobalSum(uint32_t &u){
   int ierr=MPI_Allreduce(MPI_IN_PLACE,&u,1,MPI_UINT32_T,MPI_SUM,communicator);
   assert(ierr==0);
diff --git a/lib/qcd/action/fermion/CayleyFermion5D.cc b/lib/qcd/action/fermion/CayleyFermion5D.cc
index 838b1c3d..eace6484 100644
--- a/lib/qcd/action/fermion/CayleyFermion5D.cc
+++ b/lib/qcd/action/fermion/CayleyFermion5D.cc
@@ -77,7 +77,6 @@ void CayleyFermion5D<Impl>::DminusDag(const FermionField &psi, FermionField &chi
   }
 }
 
-
 template<class Impl> void CayleyFermion5D<Impl>::CayleyReport(void)
 {
   this->Report();
@@ -119,7 +118,6 @@ template<class Impl> void CayleyFermion5D<Impl>::CayleyZeroCounters(void)
   MooeeInvTime=0;
 }
 
-
 template<class Impl>  
 void CayleyFermion5D<Impl>::M5D   (const FermionField &psi, FermionField &chi)
 {

From 1e54882f7145bd38db8ac1681cb7d4f9bceb2297 Mon Sep 17 00:00:00 2001
From: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Wed, 4 Oct 2017 10:51:06 +0100
Subject: [PATCH 103/115] Stagger

---
 tests/solver/Test_staggered_cg_prec.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/solver/Test_staggered_cg_prec.cc b/tests/solver/Test_staggered_cg_prec.cc
index 66f11d3d..0a803c21 100644
--- a/tests/solver/Test_staggered_cg_prec.cc
+++ b/tests/solver/Test_staggered_cg_prec.cc
@@ -83,5 +83,14 @@ int main (int argc, char ** argv)
   ConjugateGradient<FermionField> CG(1.0e-8,10000);
   CG(HermOpEO,src_o,res_o);
 
+  FermionField tmp(&RBGrid);
+
+  HermOpEO.Mpc(res_o,tmp);
+  std::cout << "check Mpc resid " << axpy_norm(tmp,-1.0,src_o,tmp)/norm2(src_o) << "\n";
+
+  RealD n1,n2;
+  HermOpEO.MpcDagMpc(res_o,tmp,n1,n2);
+  std::cout << "check MpcDagMpc resid " << axpy_norm(tmp,-1.0,src_o,tmp)/norm2(src_o) << "\n";
+
   Grid_finalize();
 }

From 07009c569a206b9e633e5ab01bdef386f10050c5 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 9 Oct 2017 23:16:51 +0100
Subject: [PATCH 104/115] Comms splitting improvements

---
 lib/communicator/Communicator_base.cc | 36 ++++++++++++++++-----------
 lib/communicator/Communicator_base.h  | 17 +++++++++++++
 lib/communicator/Communicator_mpi.cc  | 17 +++++++++++++
 lib/communicator/Communicator_none.cc |  4 +++
 4 files changed, 60 insertions(+), 14 deletions(-)

diff --git a/lib/communicator/Communicator_base.cc b/lib/communicator/Communicator_base.cc
index bcf429ab..ce9a3cf0 100644
--- a/lib/communicator/Communicator_base.cc
+++ b/lib/communicator/Communicator_base.cc
@@ -117,32 +117,40 @@ CartesianCommunicator::CartesianCommunicator(const std::vector<int> &processors,
   int Nchild = Nparent/childsize;
   assert (childsize * Nchild == Nparent);
 
-  int prank;  MPI_Comm_rank(parent.communicator,&prank);
-  int crank = prank % childsize;
-  int ccomm = prank / childsize;
+  std::vector<int> ccoor(_ndimension); // coor within subcommunicator
+  std::vector<int> scoor(_ndimension); // coor of split within parent
+  std::vector<int> ssize(_ndimension); // coor of split within parent
+
+  for(int d=0;d<_ndimension;d++){
+    ccoor[d] = parent._processor_coor[d] % processors[d];
+    scoor[d] = parent._processor_coor[d] / processors[d];
+    ssize[d] = parent._processors[d]/ processors[d];
+  }
+  int crank,srank;  // rank within subcomm ; rank of subcomm within blocks of subcomms
+  Lexicographic::IndexFromCoor(ccoor,crank,processors);
+  Lexicographic::IndexFromCoor(scoor,srank,ssize);
 
   MPI_Comm comm_split;
   if ( Nchild > 1 ) { 
 
-    std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec<<std::endl;
-    std::cout << GridLogMessage<<" parent grid["<< parent._ndimension<<"]    ";
-    for(int d=0;d<parent._processors.size();d++)  std::cout << parent._processors[d] << " ";
-    std::cout<<std::endl;
+    //    std::cout << GridLogMessage<<"Child communicator of "<< std::hex << parent.communicator << std::dec<<std::endl;
+    //    std::cout << GridLogMessage<<" parent grid["<< parent._ndimension<<"]    ";
+    //    for(int d=0;d<parent._processors.size();d++)  std::cout << parent._processors[d] << " ";
+    //    std::cout<<std::endl;
 
-    std::cout << GridLogMessage<<" child grid["<< _ndimension <<"]    ";
-    for(int d=0;d<processors.size();d++)  std::cout << processors[d] << " ";
-    std::cout<<std::endl;
+    //    std::cout << GridLogMessage<<" child grid["<< _ndimension <<"]    ";
+    //    for(int d=0;d<processors.size();d++)  std::cout << processors[d] << " ";
+    //    std::cout<<std::endl;
 
-    int ierr= MPI_Comm_split(parent.communicator, ccomm,crank,&comm_split);
+    int ierr= MPI_Comm_split(parent.communicator,srank,crank,&comm_split);
     assert(ierr==0);
     //////////////////////////////////////////////////////////////////////////////////////////////////////
     // Declare victory
     //////////////////////////////////////////////////////////////////////////////////////////////////////
-    std::cout << GridLogMessage<<"Divided communicator "<< parent._Nprocessors<<" into "
-	      <<Nchild <<" communicators with " << childsize << " ranks"<<std::endl;
+    //    std::cout << GridLogMessage<<"Divided communicator "<< parent._Nprocessors<<" into "
+    // 	      << Nchild <<" communicators with " << childsize << " ranks"<<std::endl;
   } else {
     comm_split=parent.communicator;
-    //    std::cout << "Passed parental communicator to a new communicator" <<std::endl;
   }
 
   //////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/lib/communicator/Communicator_base.h b/lib/communicator/Communicator_base.h
index bfdb0da1..8ff22dbd 100644
--- a/lib/communicator/Communicator_base.h
+++ b/lib/communicator/Communicator_base.h
@@ -262,6 +262,23 @@ class CartesianCommunicator {
   // Broadcast a buffer and composite larger
   ////////////////////////////////////////////////////////////
   void Broadcast(int root,void* data, int bytes);
+
+  ////////////////////////////////////////////////////////////
+  // All2All down one dimension
+  ////////////////////////////////////////////////////////////
+  template<class T> void AllToAll(int dim,std::vector<T> &in, std::vector<T> &out){
+    assert(dim>=0);
+    assert(dim<_ndimension);
+    int numnode = _processors[dim];
+    //    std::cerr << " AllToAll in.size()  "<<in.size()<<std::endl;
+    //    std::cerr << " AllToAll out.size() "<<out.size()<<std::endl;
+    assert(in.size()==out.size());
+    size_t bytes=(in.size()*sizeof(T))/numnode;
+    assert((bytes*numnode) == in.size()*sizeof(T));
+    AllToAll(dim,(void *)&in[0],(void *)&out[0],bytes);
+  }
+  void AllToAll(int dim  ,void *in,void *out,int bytes);
+  void AllToAll(void  *in,void *out,int bytes);
   
   template<class obj> void Broadcast(int root,obj &data)
     {
diff --git a/lib/communicator/Communicator_mpi.cc b/lib/communicator/Communicator_mpi.cc
index a55c0164..678e4517 100644
--- a/lib/communicator/Communicator_mpi.cc
+++ b/lib/communicator/Communicator_mpi.cc
@@ -187,6 +187,21 @@ void CartesianCommunicator::Broadcast(int root,void* data, int bytes)
 		     root,
 		     communicator);
   assert(ierr==0);
+}
+void CartesianCommunicator::AllToAll(int dim,void  *in,void *out,int bytes)
+{
+  std::vector<int> row(_ndimension,1);
+  assert(dim>=0 && dim<_ndimension);
+
+  //  Split the communicator
+  row[dim] = _processors[dim];
+
+  CartesianCommunicator Comm(row,*this);
+  Comm.AllToAll(in,out,bytes);
+}
+void CartesianCommunicator::AllToAll(void  *in,void *out,int bytes)
+{
+  MPI_Alltoall(in ,bytes,MPI_BYTE,out,bytes,MPI_BYTE,communicator);
 }
   ///////////////////////////////////////////////////////
   // Should only be used prior to Grid Init finished.
@@ -207,5 +222,7 @@ void CartesianCommunicator::BroadcastWorld(int root,void* data, int bytes)
   assert(ierr==0);
 }
 
+
+
 }
 
diff --git a/lib/communicator/Communicator_none.cc b/lib/communicator/Communicator_none.cc
index 40feefec..e9d71a15 100644
--- a/lib/communicator/Communicator_none.cc
+++ b/lib/communicator/Communicator_none.cc
@@ -98,6 +98,10 @@ void CartesianCommunicator::SendToRecvFromComplete(std::vector<CommsRequest_t> &
 {
   assert(0);
 }
+void CartesianCommunicator::AllToAll(int dim,void  *in,void *out,int bytes)
+{
+  bcopy(in,out,bytes);
+}
 
 int  CartesianCommunicator::RankWorld(void){return 0;}
 void CartesianCommunicator::Barrier(void){}

From f7cbf82c0487be5a7be37fd6b7be148b74029884 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 9 Oct 2017 23:18:48 +0100
Subject: [PATCH 105/115] Better stdout/err debug

---
 lib/util/Init.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/util/Init.cc b/lib/util/Init.cc
index 3232d32f..1266d34d 100644
--- a/lib/util/Init.cc
+++ b/lib/util/Init.cc
@@ -243,6 +243,12 @@ void Grid_init(int *argc,char ***argv)
     fname<<CartesianCommunicator::RankWorld();
     fp=freopen(fname.str().c_str(),"w",stdout);
     assert(fp!=(FILE *)NULL);
+
+    std::ostringstream ename;
+    ename<<"Grid.stderr.";
+    ename<<CartesianCommunicator::RankWorld();
+    fp=freopen(ename.str().c_str(),"w",stderr);
+    assert(fp!=(FILE *)NULL);
   }
 
   ////////////////////////////////////

From 08ca338875e90ff3c6fd246a44a68f86800f6075 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 9 Oct 2017 23:19:45 +0100
Subject: [PATCH 106/115] Split grid communication

---
 lib/lattice/Lattice_transfer.h | 301 +++++++++++++++++++++++++++++++++
 1 file changed, 301 insertions(+)

diff --git a/lib/lattice/Lattice_transfer.h b/lib/lattice/Lattice_transfer.h
index cbf31f86..713a8788 100644
--- a/lib/lattice/Lattice_transfer.h
+++ b/lib/lattice/Lattice_transfer.h
@@ -684,6 +684,307 @@ void precisionChange(Lattice<VobjOut> &out, const Lattice<VobjIn> &in){
     merge(out._odata[out_oidx], ptrs, 0);
   }
 }
+
+////////////////////////////////////////////////////////////////////////////////
+// Communicate between grids
+////////////////////////////////////////////////////////////////////////////////
+//
+// All to all plan
+//
+// Subvolume on fine grid is v.    Vectors a,b,c,d 
+//
+///////////////////////////////////////////////////////////////////////////////////////////////////////////
+// SIMPLEST CASE:
+///////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Mesh of nodes (2) ; subdivide to  1 subdivisions
+//
+// Lex ord:   
+//          N0 va0 vb0  N1 va1 vb1 
+//
+// For each dimension do an all to all
+//
+// full AllToAll(0)
+//          N0 va0 va1    N1 vb0 vb1
+//
+// REARRANGE
+//          N0 va01       N1 vb01
+//
+// Must also rearrange data to get into the NEW lex order of grid at each stage. Some kind of "insert/extract".
+// NB: Easiest to programme if keep in lex order.
+//
+///////////////////////////////////////////////////////////////////////////////////////////////////////////
+// SIMPLE CASE:
+///////////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// Mesh of nodes (2x2) ; subdivide to  1x1 subdivisions
+//
+// Lex ord:   
+//          N0 va0 vb0 vc0 vd0       N1 va1 vb1 vc1 vd1  
+//          N2 va2 vb2 vc2 vd2       N3 va3 vb3 vc3 vd3 
+//
+// Ratio = full[dim] / split[dim]
+//
+// For each dimension do an all to all; get Nvec -> Nvec / ratio
+//                                          Ldim -> Ldim * ratio
+//                                          LocalVol -> LocalVol * ratio
+// full AllToAll(0)
+//          N0 va0 vb0 va1 vb1       N1 vc0 vd0 vc1 vd1   
+//          N2 va2 vb2 va3 vb3       N3 vc2 vd2 vc3 vd3 
+//
+// REARRANGE
+//          N0 va01 vb01      N1 vc01 vd01
+//          N2 va23 vb23      N3 vc23 vd23
+//
+// full AllToAll(1)           // Not what is wanted. FIXME
+//          N0 va01 va23      N1 vc01 vc23 
+//          N2 vb01 vb23      N3 vd01 vd23
+// 
+// REARRANGE
+//          N0 va0123      N1 vc0123
+//          N2 vb0123      N3 vd0123
+//
+// Must also rearrange data to get into the NEW lex order of grid at each stage. Some kind of "insert/extract".
+// NB: Easiest to programme if keep in lex order.
+//
+/////////////////////////////////////////////////////////
+template<class Vobj>
+void Grid_split(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
+{
+  typedef typename Vobj::scalar_object Sobj;
+
+  int full_vecs   = full.size();
+
+  assert(full_vecs>=1);
+
+  GridBase * full_grid = full[0]._grid;
+  GridBase *split_grid = split._grid;
+
+  int       ndim  = full_grid->_ndimension;
+  int  full_nproc = full_grid->_Nprocessors;
+  int split_nproc =split_grid->_Nprocessors;
+
+  ////////////////////////////////
+  // Checkerboard management
+  ////////////////////////////////
+  int cb = full[0].checkerboard;
+  split.checkerboard = cb;
+
+  //////////////////////////////
+  // Checks
+  //////////////////////////////
+  assert(full_grid->_ndimension==split_grid->_ndimension);
+  for(int n=0;n<full_vecs;n++){
+    assert(full[n].checkerboard == cb);
+    for(int d=0;d<ndim;d++){
+      assert(full[n]._grid->_gdimensions[d]==split._grid->_gdimensions[d]);
+      assert(full[n]._grid->_fdimensions[d]==split._grid->_fdimensions[d]);
+    }
+  }
+
+  int   nvector   =full_nproc/split_nproc; 
+  assert(nvector*split_nproc==full_nproc);
+  assert(nvector == full_vecs);
+
+  std::vector<int> ratio(ndim);
+  for(int d=0;d<ndim;d++){
+    ratio[d] = full_grid->_processors[d]/ split_grid->_processors[d];
+  }
+
+  int lsites = full_grid->lSites();
+  Integer sz = lsites * nvector;
+  std::vector<Sobj> tmpdata(sz);
+  std::vector<Sobj> alldata(sz);
+  std::vector<Sobj> scalardata(lsites); 
+  for(int v=0;v<nvector;v++){
+    unvectorizeToLexOrdArray(scalardata,full[v]);    
+    parallel_for(int site=0;site<lsites;site++){
+      alldata[v*lsites+site] = scalardata[site];
+    }
+  }
+
+  int nvec = nvector; // Counts down to 1 as we collapse dims
+  std::vector<int> ldims = full_grid->_ldimensions;
+  std::vector<int> lcoor(ndim);
+
+  for(int d=0;d<ndim;d++){
+
+    if ( ratio[d] != 1 ) {
+
+      full_grid ->AllToAll(d,alldata,tmpdata);
+
+      //////////////////////////////////////////
+      //Local volume for this dimension is expanded by ratio of processor extents
+      // Number of vectors is decreased by same factor
+      // Rearrange to lexico for bigger volume
+      //////////////////////////////////////////
+      nvec    /= ratio[d];
+      auto rdims = ldims; rdims[d]  *=   ratio[d];
+      auto rsites= lsites*ratio[d];
+      for(int v=0;v<nvec;v++){
+
+	// For loop over each site within old subvol
+	for(int lsite=0;lsite<lsites;lsite++){
+
+	  Lexicographic::CoorFromIndex(lcoor, lsite, ldims);	  
+
+	  for(int r=0;r<ratio[d];r++){ // ratio*nvec terms
+
+	    auto rcoor = lcoor;	    rcoor[d]  += r*ldims[d];
+
+	    int rsite; Lexicographic::IndexFromCoor(rcoor, rsite, rdims);	  
+	    rsite += v * rsites;
+
+	    int rmul=nvec*lsites;
+	    int vmul=     lsites;
+	    alldata[rsite] = tmpdata[lsite+r*rmul+v*vmul];
+
+	  }
+	}
+      }
+      ldims[d]*= ratio[d];
+      lsites  *= ratio[d];
+
+      if ( split_grid->_processors[d] > 1 ) {
+	tmpdata = alldata;
+	split_grid->AllToAll(d,tmpdata,alldata);
+      }
+    }
+  }
+
+  vectorizeFromLexOrdArray(alldata,split);    
+}
+
+template<class Vobj>
+void Grid_split(Lattice<Vobj> &full,Lattice<Vobj>   & split)
+{
+  int nvector = full._grid->_Nprocessors / split._grid->_Nprocessors;
+  std::vector<Lattice<Vobj> > full_v(nvector,full._grid);
+  for(int n=0;n<nvector;n++){
+    full_v[n] = full;
+  }
+  Grid_split(full_v,split);
+}
+
+template<class Vobj>
+void Grid_unsplit(std::vector<Lattice<Vobj> > & full,Lattice<Vobj>   & split)
+{
+  typedef typename Vobj::scalar_object Sobj;
+
+  int full_vecs   = full.size();
+
+  assert(full_vecs>=1);
+
+  GridBase * full_grid = full[0]._grid;
+  GridBase *split_grid = split._grid;
+
+  int       ndim  = full_grid->_ndimension;
+  int  full_nproc = full_grid->_Nprocessors;
+  int split_nproc =split_grid->_Nprocessors;
+
+  ////////////////////////////////
+  // Checkerboard management
+  ////////////////////////////////
+  int cb = full[0].checkerboard;
+  split.checkerboard = cb;
+
+  //////////////////////////////
+  // Checks
+  //////////////////////////////
+  assert(full_grid->_ndimension==split_grid->_ndimension);
+  for(int n=0;n<full_vecs;n++){
+    assert(full[n].checkerboard == cb);
+    for(int d=0;d<ndim;d++){
+      assert(full[n]._grid->_gdimensions[d]==split._grid->_gdimensions[d]);
+      assert(full[n]._grid->_fdimensions[d]==split._grid->_fdimensions[d]);
+    }
+  }
+
+  int   nvector   =full_nproc/split_nproc; 
+  assert(nvector*split_nproc==full_nproc);
+  assert(nvector == full_vecs);
+
+  std::vector<int> ratio(ndim);
+  for(int d=0;d<ndim;d++){
+    ratio[d] = full_grid->_processors[d]/ split_grid->_processors[d];
+  }
+
+  int lsites = full_grid->lSites();
+  Integer sz = lsites * nvector;
+  std::vector<Sobj> tmpdata(sz);
+  std::vector<Sobj> alldata(sz);
+  std::vector<Sobj> scalardata(lsites); 
+
+  unvectorizeToLexOrdArray(alldata,split);    
+
+  /////////////////////////////////////////////////////////////////
+  // Start from split grid and work towards full grid
+  /////////////////////////////////////////////////////////////////
+  std::vector<int> lcoor(ndim);
+  std::vector<int> rcoor(ndim);
+
+  int nvec = 1;
+  lsites = split_grid->lSites();
+  std::vector<int> ldims = split_grid->_ldimensions;
+
+  for(int d=ndim-1;d>=0;d--){
+
+    if ( ratio[d] != 1 ) {
+
+      if ( split_grid->_processors[d] > 1 ) {
+	tmpdata = alldata;
+	split_grid->AllToAll(d,tmpdata,alldata);
+      }
+
+      //////////////////////////////////////////
+      //Local volume for this dimension is expanded by ratio of processor extents
+      // Number of vectors is decreased by same factor
+      // Rearrange to lexico for bigger volume
+      //////////////////////////////////////////
+      auto rsites= lsites/ratio[d];
+      auto rdims = ldims; rdims[d]/=ratio[d];
+
+      for(int v=0;v<nvec;v++){
+
+	// rsite, rcoor --> smaller local volume
+	// lsite, lcoor --> bigger original (single node?) volume
+	// For loop over each site within smaller subvol
+	for(int rsite=0;rsite<rsites;rsite++){
+
+	  Lexicographic::CoorFromIndex(rcoor, rsite, rdims);	  
+	  int lsite;
+
+	  for(int r=0;r<ratio[d];r++){ 
+
+	    lcoor = rcoor; lcoor[d] += r*rdims[d];
+	    Lexicographic::IndexFromCoor(lcoor, lsite, ldims); lsite += v * lsites;
+
+	    int rmul=nvec*rsites;
+	    int vmul=     rsites;
+	    tmpdata[rsite+r*rmul+v*vmul]=alldata[lsite];
+
+	  }
+	}
+      }
+      nvec   *= ratio[d];
+      ldims[d]=rdims[d];
+      lsites  =rsites;
+
+      full_grid ->AllToAll(d,tmpdata,alldata);
+    }
+  }
+
+  lsites = full_grid->lSites();
+  for(int v=0;v<nvector;v++){
+    parallel_for(int site=0;site<lsites;site++){
+      scalardata[site] = alldata[v*lsites+site];
+    }
+    assert(v<full.size());
+
+    vectorizeFromLexOrdArray(scalardata,full[v]);    
+
+  }
+}
+
  
 }
 #endif

From 153672d8ec22f368cdb10b9414d8727f6e198ca0 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Mon, 9 Oct 2017 23:20:58 +0100
Subject: [PATCH 107/115] Split CG testing

---
 tests/solver/Test_dwf_mrhs_cg.cc       |  64 +++++++---
 tests/solver/Test_dwf_mrhs_cg_mpi.cc   | 144 ++++++++++++++++++++++
 tests/solver/Test_dwf_mrhs_cg_mpieo.cc | 163 +++++++++++++++++++++++++
 3 files changed, 356 insertions(+), 15 deletions(-)
 create mode 100644 tests/solver/Test_dwf_mrhs_cg_mpi.cc
 create mode 100644 tests/solver/Test_dwf_mrhs_cg_mpieo.cc

diff --git a/tests/solver/Test_dwf_mrhs_cg.cc b/tests/solver/Test_dwf_mrhs_cg.cc
index 2d2cfcb1..d9215db2 100644
--- a/tests/solver/Test_dwf_mrhs_cg.cc
+++ b/tests/solver/Test_dwf_mrhs_cg.cc
@@ -38,7 +38,7 @@ int main (int argc, char ** argv)
   typedef typename DomainWallFermionR::ComplexField ComplexField; 
   typename DomainWallFermionR::ImplParams params; 
 
-  const int Ls=8;
+  const int Ls=4;
 
   Grid_init(&argc,&argv);
 
@@ -47,29 +47,24 @@ int main (int argc, char ** argv)
   std::vector<int> mpi_layout  = GridDefaultMpi();
   std::vector<int> mpi_split (mpi_layout.size(),1);
 
-  std::cout << "UGrid (world root)"<<std::endl;
   GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
-
-  std::cout << "FGrid (child of UGrid)"<<std::endl;
   GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * rbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
 
   int nrhs = UGrid->RankCount() ;
 
   /////////////////////////////////////////////
   // Split into 1^4 mpi communicators
   /////////////////////////////////////////////
-  std::cout << "SGrid (world root)"<<std::endl;
   GridCartesian         * SGrid = new GridCartesian(GridDefaultLatt(),
 						    GridDefaultSimd(Nd,vComplex::Nsimd()),
 						    mpi_split,
 						    *UGrid); 
 
   GridCartesian         * SFGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,SGrid);
-  std::cout << "SFGrid"<<std::endl;
   GridRedBlackCartesian * SrbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(SGrid);
-  std::cout << "SrbGrid"<<std::endl;
   GridRedBlackCartesian * SFrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,SGrid);
-  std::cout << "SFrbGrid"<<std::endl;
 
   ///////////////////////////////////////////////
   // Set up the problem as a 4d spreadout job
@@ -79,10 +74,12 @@ int main (int argc, char ** argv)
   GridParallelRNG pRNG(UGrid );  pRNG.SeedFixedIntegers(seeds);
   GridParallelRNG pRNG5(FGrid);  pRNG5.SeedFixedIntegers(seeds);
   std::vector<FermionField>    src(nrhs,FGrid);
+  std::vector<FermionField> src_chk(nrhs,FGrid);
   std::vector<FermionField> result(nrhs,FGrid);
+  FermionField tmp(FGrid);
 
   for(int s=0;s<nrhs;s++) random(pRNG5,src[s]);
-  for(int s=0;s<nrhs;s++) result[s] = zero;
+  for(int s=0;s<nrhs;s++) result[s]=zero;
 
   LatticeGaugeField Umu(UGrid); SU3::HotConfiguration(pRNG,Umu);
 
@@ -99,6 +96,8 @@ int main (int argc, char ** argv)
   int me = UGrid->ThisRank();
   LatticeGaugeField s_Umu(SGrid);
   FermionField s_src(SFGrid);
+  FermionField s_src_split(SFGrid);
+  FermionField s_tmp(SFGrid);
   FermionField s_res(SFGrid);
 
   {
@@ -157,6 +156,24 @@ int main (int argc, char ** argv)
     FGrid->Barrier();
   }
 
+  ///////////////////////////////////////////////////////////////
+  // split the source out using MPI instead of I/O
+  ///////////////////////////////////////////////////////////////
+  std::cout << GridLogMessage << " Splitting the grid data "<<std::endl;
+  Grid_split  (src,s_src_split);
+  std::cout << GridLogMessage << " Finished splitting the grid data "<<std::endl;
+  for(int n=0;n<nrhs;n++){
+    std::cout <<GridLogMessage<<"Full "<< n <<" "<< norm2(src[n])<<std::endl;
+  }
+  s_tmp = s_src_split - s_src;
+  for(int n=0;n<nrhs;n++){
+    FGrid->Barrier();
+    if ( n==me ) {
+      std::cerr << GridLogMessage<<"Split "<< me << " " << norm2(s_src_split) << " " << norm2(s_src)<< " diff " << norm2(s_tmp)<<std::endl;
+    }
+    FGrid->Barrier();
+  }
+
 
   ///////////////////////////////////////////////////////////////
   // Set up N-solvers as trivially parallel
@@ -164,6 +181,7 @@ int main (int argc, char ** argv)
 
   RealD mass=0.01;
   RealD M5=1.8;
+  DomainWallFermionR Dchk(Umu,*FGrid,*FrbGrid,*UGrid,*rbGrid,mass,M5);
   DomainWallFermionR Ddwf(s_Umu,*SFGrid,*SFrbGrid,*SGrid,*SrbGrid,mass,M5);
 
   std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
@@ -171,25 +189,41 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
 
   MdagMLinearOperator<DomainWallFermionR,FermionField> HermOp(Ddwf);
+  MdagMLinearOperator<DomainWallFermionR,FermionField> HermOpCk(Dchk);
   ConjugateGradient<FermionField> CG((1.0e-8/(me+1)),10000);
   s_res = zero;
   CG(HermOp,s_src,s_res);
 
-  ///////////////////////////////////////
-  // Share the information
-  ///////////////////////////////////////
+  /////////////////////////////////////////////////////////////
+  // Report how long they all took
+  /////////////////////////////////////////////////////////////
   std::vector<uint32_t> iterations(nrhs,0);
   iterations[me] = CG.IterationsToComplete;
 
   for(int n=0;n<nrhs;n++){
     UGrid->GlobalSum(iterations[n]);
+    std::cout << GridLogMessage<<" Rank "<<n<<" "<< iterations[n]<<" CG iterations"<<std::endl;
   }
 
   /////////////////////////////////////////////////////////////
-  // Report how long they all took
+  // Gather and residual check on the results
   /////////////////////////////////////////////////////////////
-  for(int r=0;r<nrhs;r++){
-    std::cout << GridLogMessage<<" Rank "<<r<<" "<< iterations[r]<<" CG iterations"<<std::endl;
+  std::cout << GridLogMessage<< "Unsplitting the result"<<std::endl;
+  Grid_unsplit(result,s_res);
+  /*
+  Grid_unsplit(src_chk,s_src);
+  for(int n=0;n<nrhs;n++){
+    tmp = src[n]-src_chk[n];
+    std::cout << " src_chk "<<n<<" "<<norm2(src_chk[n])<<" " <<norm2(src[n])<<" " <<norm2(tmp)<< std::endl;
+    std::cout << " diff " <<tmp<<std::endl;
   }
+  */
+
+  std::cout << GridLogMessage<< "Checking the residuals"<<std::endl;
+  for(int n=0;n<nrhs;n++){
+    HermOpCk.HermOp(result[n],tmp); tmp = tmp - src[n];
+    std::cout << GridLogMessage<<" resid["<<n<<"]  "<< norm2(tmp)<<std::endl;
+  }
+
   Grid_finalize();
 }
diff --git a/tests/solver/Test_dwf_mrhs_cg_mpi.cc b/tests/solver/Test_dwf_mrhs_cg_mpi.cc
new file mode 100644
index 00000000..90969b85
--- /dev/null
+++ b/tests/solver/Test_dwf_mrhs_cg_mpi.cc
@@ -0,0 +1,144 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_dwf_mrhs_cg.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+#include <Grid/algorithms/iterative/BlockConjugateGradient.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+int main (int argc, char ** argv)
+{
+  typedef typename DomainWallFermionR::FermionField FermionField; 
+  typedef typename DomainWallFermionR::ComplexField ComplexField; 
+  typename DomainWallFermionR::ImplParams params; 
+
+  const int Ls=4;
+
+  Grid_init(&argc,&argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+  std::vector<int> mpi_split (mpi_layout.size(),1);
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * rbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  int nrhs = UGrid->RankCount() ;
+
+  /////////////////////////////////////////////
+  // Split into 1^4 mpi communicators
+  /////////////////////////////////////////////
+  GridCartesian         * SGrid = new GridCartesian(GridDefaultLatt(),
+						    GridDefaultSimd(Nd,vComplex::Nsimd()),
+						    mpi_split,
+						    *UGrid); 
+
+  GridCartesian         * SFGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,SGrid);
+  GridRedBlackCartesian * SrbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(SGrid);
+  GridRedBlackCartesian * SFrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,SGrid);
+
+  ///////////////////////////////////////////////
+  // Set up the problem as a 4d spreadout job
+  ///////////////////////////////////////////////
+  std::vector<int> seeds({1,2,3,4});
+
+  GridParallelRNG pRNG(UGrid );  pRNG.SeedFixedIntegers(seeds);
+  GridParallelRNG pRNG5(FGrid);  pRNG5.SeedFixedIntegers(seeds);
+  std::vector<FermionField>    src(nrhs,FGrid);
+  std::vector<FermionField> src_chk(nrhs,FGrid);
+  std::vector<FermionField> result(nrhs,FGrid);
+  FermionField tmp(FGrid);
+
+  for(int s=0;s<nrhs;s++) random(pRNG5,src[s]);
+  for(int s=0;s<nrhs;s++) result[s]=zero;
+
+  LatticeGaugeField Umu(UGrid); SU3::HotConfiguration(pRNG,Umu);
+
+  /////////////////
+  // MPI only sends
+  /////////////////
+  int me = UGrid->ThisRank();
+
+  LatticeGaugeField s_Umu(SGrid);
+  FermionField s_src(SFGrid);
+  FermionField s_tmp(SFGrid);
+  FermionField s_res(SFGrid);
+
+  ///////////////////////////////////////////////////////////////
+  // split the source out using MPI instead of I/O
+  ///////////////////////////////////////////////////////////////
+  Grid_split  (Umu,s_Umu);
+  Grid_split  (src,s_src);
+
+  ///////////////////////////////////////////////////////////////
+  // Set up N-solvers as trivially parallel
+  ///////////////////////////////////////////////////////////////
+  RealD mass=0.01;
+  RealD M5=1.8;
+  DomainWallFermionR Dchk(Umu,*FGrid,*FrbGrid,*UGrid,*rbGrid,mass,M5);
+  DomainWallFermionR Ddwf(s_Umu,*SFGrid,*SFrbGrid,*SGrid,*SrbGrid,mass,M5);
+
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+  std::cout << GridLogMessage << " Calling DWF CG "<<std::endl;
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+
+  MdagMLinearOperator<DomainWallFermionR,FermionField> HermOp(Ddwf);
+  MdagMLinearOperator<DomainWallFermionR,FermionField> HermOpCk(Dchk);
+  ConjugateGradient<FermionField> CG((1.0e-8/(me+1)),10000);
+  s_res = zero;
+  CG(HermOp,s_src,s_res);
+
+  /////////////////////////////////////////////////////////////
+  // Report how long they all took
+  /////////////////////////////////////////////////////////////
+  std::vector<uint32_t> iterations(nrhs,0);
+  iterations[me] = CG.IterationsToComplete;
+
+  for(int n=0;n<nrhs;n++){
+    UGrid->GlobalSum(iterations[n]);
+    std::cout << GridLogMessage<<" Rank "<<n<<" "<< iterations[n]<<" CG iterations"<<std::endl;
+  }
+
+  /////////////////////////////////////////////////////////////
+  // Gather and residual check on the results
+  /////////////////////////////////////////////////////////////
+  std::cout << GridLogMessage<< "Unsplitting the result"<<std::endl;
+  Grid_unsplit(result,s_res);
+
+  std::cout << GridLogMessage<< "Checking the residuals"<<std::endl;
+  for(int n=0;n<nrhs;n++){
+    HermOpCk.HermOp(result[n],tmp); tmp = tmp - src[n];
+    std::cout << GridLogMessage<<" resid["<<n<<"]  "<< norm2(tmp)<<std::endl;
+  }
+
+  Grid_finalize();
+}
diff --git a/tests/solver/Test_dwf_mrhs_cg_mpieo.cc b/tests/solver/Test_dwf_mrhs_cg_mpieo.cc
new file mode 100644
index 00000000..14115b59
--- /dev/null
+++ b/tests/solver/Test_dwf_mrhs_cg_mpieo.cc
@@ -0,0 +1,163 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_dwf_mrhs_cg.cc
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+#include <Grid/algorithms/iterative/BlockConjugateGradient.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+int main (int argc, char ** argv)
+{
+  typedef typename DomainWallFermionR::FermionField FermionField; 
+  typedef typename DomainWallFermionR::ComplexField ComplexField; 
+  typename DomainWallFermionR::ImplParams params; 
+
+  const int Ls=4;
+
+  Grid_init(&argc,&argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+  std::vector<int> mpi_split (mpi_layout.size(),1);
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * rbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  int nrhs = UGrid->RankCount() ;
+
+  /////////////////////////////////////////////
+  // Split into 1^4 mpi communicators
+  /////////////////////////////////////////////
+  GridCartesian         * SGrid = new GridCartesian(GridDefaultLatt(),
+						    GridDefaultSimd(Nd,vComplex::Nsimd()),
+						    mpi_split,
+						    *UGrid); 
+
+  GridCartesian         * SFGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,SGrid);
+  GridRedBlackCartesian * SrbGrid  = SpaceTimeGrid::makeFourDimRedBlackGrid(SGrid);
+  GridRedBlackCartesian * SFrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,SGrid);
+
+  ///////////////////////////////////////////////
+  // Set up the problem as a 4d spreadout job
+  ///////////////////////////////////////////////
+  std::vector<int> seeds({1,2,3,4});
+
+  GridParallelRNG pRNG(UGrid );  pRNG.SeedFixedIntegers(seeds);
+  GridParallelRNG pRNG5(FGrid);  pRNG5.SeedFixedIntegers(seeds);
+  std::vector<FermionField>    src(nrhs,FGrid);
+  std::vector<FermionField> src_chk(nrhs,FGrid);
+  std::vector<FermionField> result(nrhs,FGrid);
+  FermionField tmp(FGrid);
+
+  std::vector<FermionField> src_e(nrhs,FrbGrid);
+  std::vector<FermionField> src_o(nrhs,FrbGrid);
+
+  for(int s=0;s<nrhs;s++) random(pRNG5,src[s]);
+  for(int s=0;s<nrhs;s++) result[s]=zero;
+
+  LatticeGaugeField Umu(UGrid); SU3::HotConfiguration(pRNG,Umu);
+
+  /////////////////
+  // MPI only sends
+  /////////////////
+  int me = UGrid->ThisRank();
+
+  LatticeGaugeField s_Umu(SGrid);
+  FermionField s_src(SFGrid);
+  FermionField s_src_e(SFrbGrid);
+  FermionField s_src_o(SFrbGrid);
+  FermionField s_tmp(SFGrid);
+  FermionField s_res(SFGrid);
+
+  ///////////////////////////////////////////////////////////////
+  // split the source out using MPI instead of I/O
+  ///////////////////////////////////////////////////////////////
+  Grid_split  (Umu,s_Umu);
+  Grid_split  (src,s_src);
+
+  ///////////////////////////////////////////////////////////////
+  // Check even odd cases
+  ///////////////////////////////////////////////////////////////
+  for(int s=0;s<nrhs;s++){
+    pickCheckerboard(Odd , src_o[s], src[s]);
+    pickCheckerboard(Even, src_e[s], src[s]);
+  }
+  Grid_split  (src_e,s_src_e);
+  Grid_split  (src_o,s_src_o);
+  setCheckerboard(s_tmp, s_src_o);
+  setCheckerboard(s_tmp, s_src_e);
+  s_tmp = s_tmp - s_src;
+  std::cout << GridLogMessage<<" EvenOdd Difference " <<norm2(s_tmp)<<std::endl;
+
+  ///////////////////////////////////////////////////////////////
+  // Set up N-solvers as trivially parallel
+  ///////////////////////////////////////////////////////////////
+  RealD mass=0.01;
+  RealD M5=1.8;
+  DomainWallFermionR Dchk(Umu,*FGrid,*FrbGrid,*UGrid,*rbGrid,mass,M5);
+  DomainWallFermionR Ddwf(s_Umu,*SFGrid,*SFrbGrid,*SGrid,*SrbGrid,mass,M5);
+
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+  std::cout << GridLogMessage << " Calling DWF CG "<<std::endl;
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+
+  MdagMLinearOperator<DomainWallFermionR,FermionField> HermOp(Ddwf);
+  MdagMLinearOperator<DomainWallFermionR,FermionField> HermOpCk(Dchk);
+  ConjugateGradient<FermionField> CG((1.0e-8/(me+1)),10000);
+  s_res = zero;
+  CG(HermOp,s_src,s_res);
+
+  /////////////////////////////////////////////////////////////
+  // Report how long they all took
+  /////////////////////////////////////////////////////////////
+  std::vector<uint32_t> iterations(nrhs,0);
+  iterations[me] = CG.IterationsToComplete;
+
+  for(int n=0;n<nrhs;n++){
+    UGrid->GlobalSum(iterations[n]);
+    std::cout << GridLogMessage<<" Rank "<<n<<" "<< iterations[n]<<" CG iterations"<<std::endl;
+  }
+
+  /////////////////////////////////////////////////////////////
+  // Gather and residual check on the results
+  /////////////////////////////////////////////////////////////
+  std::cout << GridLogMessage<< "Unsplitting the result"<<std::endl;
+  Grid_unsplit(result,s_res);
+
+  std::cout << GridLogMessage<< "Checking the residuals"<<std::endl;
+  for(int n=0;n<nrhs;n++){
+    HermOpCk.HermOp(result[n],tmp); tmp = tmp - src[n];
+    std::cout << GridLogMessage<<" resid["<<n<<"]  "<< norm2(tmp)<<std::endl;
+  }
+
+  Grid_finalize();
+}

From f0e084a88c98c2ebf94a7c5f9f6f145699bfd827 Mon Sep 17 00:00:00 2001
From: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 10 Oct 2017 10:00:43 +0100
Subject: [PATCH 108/115] Schur staggered

---
 lib/algorithms/LinearOperator.h          | 104 +++++++++-
 lib/algorithms/iterative/SchurRedBlack.h | 240 +++++++++++++++++++++++
 2 files changed, 342 insertions(+), 2 deletions(-)

diff --git a/lib/algorithms/LinearOperator.h b/lib/algorithms/LinearOperator.h
index 6cb77296..6e4da248 100644
--- a/lib/algorithms/LinearOperator.h
+++ b/lib/algorithms/LinearOperator.h
@@ -192,10 +192,10 @@ namespace Grid {
 	ni=Mpc(in,tmp);
 	no=MpcDag(tmp,out);
       }
-      void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
+      virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
 	MpcDagMpc(in,out,n1,n2);
       }
-      void HermOp(const Field &in, Field &out){
+      virtual void HermOp(const Field &in, Field &out){
 	RealD n1,n2;
 	HermOpAndNorm(in,out,n1,n2);
       }
@@ -300,6 +300,106 @@ namespace Grid {
       }
     };
 
+      //
+    template<class Matrix,class Field>
+      class SchurStaggeredOperator :  public SchurOperatorBase<Field> {
+    protected:
+      Matrix &_Mat;
+    public:
+      SchurStaggeredOperator (Matrix &Mat): _Mat(Mat){};
+      virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
+	ComplexD dot;
+	n2=Mpc(in,out);
+	dot= innerProduct(in,out);
+	n1= real(dot);
+      }
+      virtual void HermOp(const Field &in, Field &out){
+	Mpc(in,out);
+      }
+      virtual  RealD Mpc      (const Field &in, Field &out) {
+	Field tmp(in._grid);
+	_Mat.Meooe(in,tmp);
+	_Mat.MooeeInv(tmp,out);
+	_Mat.MeooeDag(out,tmp);
+	_Mat.Mooee(in,out);
+        return axpy_norm(out,-1.0,tmp,out);
+      }
+      virtual  RealD MpcDag   (const Field &in, Field &out){
+	return Mpc(in,out);
+      }
+      virtual void MpcDagMpc(const Field &in, Field &out,RealD &ni,RealD &no) {
+	assert(0);// Never need with staggered
+      }
+    };
+    template<class Matrix,class Field> using SchurStagOperator = SchurStaggeredOperator<Matrix,Field>;
+
+  // This is specific to (Z)mobius fermions
+  template<class Matrix, class Field>
+    class KappaSimilarityTransform {
+  public:
+
+    typedef typename Matrix::Coeff_t                     Coeff_t;
+    std::vector<Coeff_t> kappa, kappaDag, kappaInv, kappaInvDag;
+
+    KappaSimilarityTransform (Matrix &zmob) {
+      for (int i=0;i<(int)zmob.bs.size();i++) {
+	Coeff_t k = 1.0 / ( 2.0 * (zmob.bs[i] *(4 - zmob.M5) + 1.0) );
+	kappa.push_back( k );
+	kappaDag.push_back( conj(k) );
+	kappaInv.push_back( 1.0 / k );
+	kappaInvDag.push_back( 1.0 / conj(k) );
+      }
+    }
+
+  template<typename vobj>
+    void sscale(const Lattice<vobj>& in, Lattice<vobj>& out, Coeff_t* s) {
+    GridBase *grid=out._grid;
+    out.checkerboard = in.checkerboard;
+    assert(grid->_simd_layout[0] == 1); // should be fine for ZMobius for now
+    int Ls = grid->_rdimensions[0];
+    parallel_for(int ss=0;ss<grid->oSites();ss++){
+      vobj tmp = s[ss % Ls]*in._odata[ss];
+      vstream(out._odata[ss],tmp);
+    }
+  }
+
+  RealD sscale_norm(const Field& in, Field& out, Coeff_t* s) {
+    sscale(in,out,s);
+    return norm2(out);
+  }
+
+  virtual RealD M       (const Field& in, Field& out) { return sscale_norm(in,out,&kappa[0]);   }
+  virtual RealD MDag    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaDag[0]);}
+  virtual RealD MInv    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInv[0]);}
+  virtual RealD MInvDag (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInvDag[0]);}
+
+  };
+
+  template<class Matrix,class Field>
+    class SchurDiagTwoKappaOperator :  public SchurOperatorBase<Field> {
+  public:
+    KappaSimilarityTransform<Matrix, Field> _S;
+    SchurDiagTwoOperator<Matrix, Field> _Mat;
+
+    SchurDiagTwoKappaOperator (Matrix &Mat): _S(Mat), _Mat(Mat) {};
+
+    virtual  RealD Mpc      (const Field &in, Field &out) {
+      Field tmp(in._grid);
+
+      _S.MInv(in,out);
+      _Mat.Mpc(out,tmp);
+      return _S.M(tmp,out);
+
+    }
+    virtual  RealD MpcDag   (const Field &in, Field &out){
+      Field tmp(in._grid);
+
+      _S.MDag(in,out);
+      _Mat.MpcDag(out,tmp);
+      return _S.MInvDag(tmp,out);
+    }
+  };
+
 
     /////////////////////////////////////////////////////////////
     // Base classes for functions of operators
diff --git a/lib/algorithms/iterative/SchurRedBlack.h b/lib/algorithms/iterative/SchurRedBlack.h
index 5caabb4b..b6eab762 100644
--- a/lib/algorithms/iterative/SchurRedBlack.h
+++ b/lib/algorithms/iterative/SchurRedBlack.h
@@ -63,6 +63,85 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
    */
 namespace Grid {
 
+  ///////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Take a matrix and form a Red Black solver calling a Herm solver
+  // Use of RB info prevents making SchurRedBlackSolve conform to standard interface
+  ///////////////////////////////////////////////////////////////////////////////////////////////////////
+
+  template<class Field> class SchurRedBlackStaggeredSolve {
+  private:
+    OperatorFunction<Field> & _HermitianRBSolver;
+    int CBfactorise;
+  public:
+
+    /////////////////////////////////////////////////////
+    // Wrap the usual normal equations Schur trick
+    /////////////////////////////////////////////////////
+  SchurRedBlackStaggeredSolve(OperatorFunction<Field> &HermitianRBSolver)  :
+     _HermitianRBSolver(HermitianRBSolver) 
+    { 
+      CBfactorise=0;
+    };
+
+    template<class Matrix>
+      void operator() (Matrix & _Matrix,const Field &in, Field &out){
+
+      // FIXME CGdiagonalMee not implemented virtual function
+      // FIXME use CBfactorise to control schur decomp
+      GridBase *grid = _Matrix.RedBlackGrid();
+      GridBase *fgrid= _Matrix.Grid();
+
+      SchurStaggeredOperator<Matrix,Field> _HermOpEO(_Matrix);
+ 
+      Field src_e(grid);
+      Field src_o(grid);
+      Field sol_e(grid);
+      Field sol_o(grid);
+      Field   tmp(grid);
+      Field  Mtmp(grid);
+      Field resid(fgrid);
+
+      pickCheckerboard(Even,src_e,in);
+      pickCheckerboard(Odd ,src_o,in);
+      pickCheckerboard(Even,sol_e,out);
+      pickCheckerboard(Odd ,sol_o,out);
+    
+      /////////////////////////////////////////////////////
+      // src_o = Mdag * (source_o - Moe MeeInv source_e)
+      /////////////////////////////////////////////////////
+      _Matrix.MooeeInv(src_e,tmp);     assert(  tmp.checkerboard ==Even);
+      _Matrix.Meooe   (tmp,Mtmp);      assert( Mtmp.checkerboard ==Odd);     
+      tmp=src_o-Mtmp;                  assert(  tmp.checkerboard ==Odd);     
+
+      _Matrix.Mooee(tmp,src_o);     assert(src_o.checkerboard ==Odd);
+
+      //////////////////////////////////////////////////////////////
+      // Call the red-black solver
+      //////////////////////////////////////////////////////////////
+      std::cout<<GridLogMessage << "SchurRedBlack solver calling the MpcDagMp solver" <<std::endl;
+      _HermitianRBSolver(_HermOpEO,src_o,sol_o);  assert(sol_o.checkerboard==Odd);
+
+      ///////////////////////////////////////////////////
+      // sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
+      ///////////////////////////////////////////////////
+      _Matrix.Meooe(sol_o,tmp);        assert(  tmp.checkerboard   ==Even);
+      src_e = src_e-tmp;               assert(  src_e.checkerboard ==Even);
+      _Matrix.MooeeInv(src_e,sol_e);   assert(  sol_e.checkerboard ==Even);
+     
+      setCheckerboard(out,sol_e); assert(  sol_e.checkerboard ==Even);
+      setCheckerboard(out,sol_o); assert(  sol_o.checkerboard ==Odd );
+
+      // Verify the unprec residual
+      _Matrix.M(out,resid); 
+      resid = resid-in;
+      RealD ns = norm2(in);
+      RealD nr = norm2(resid);
+
+      std::cout<<GridLogMessage << "SchurRedBlackStaggered solver true unprec resid "<< std::sqrt(nr/ns) <<" nr "<< nr <<" ns "<<ns << std::endl;
+    }     
+  };
+  template<class Field> using SchurRedBlackStagSolve = SchurRedBlackStaggeredSolve<Field>;
+
   ///////////////////////////////////////////////////////////////////////////////////////////////////////
   // Take a matrix and form a Red Black solver calling a Herm solver
   // Use of RB info prevents making SchurRedBlackSolve conform to standard interface
@@ -141,5 +220,166 @@ namespace Grid {
     }     
   };
 
+
+  ///////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Take a matrix and form a Red Black solver calling a Herm solver
+  // Use of RB info prevents making SchurRedBlackSolve conform to standard interface
+  ///////////////////////////////////////////////////////////////////////////////////////////////////////
+  template<class Field> class SchurRedBlackDiagTwoSolve {
+  private:
+    OperatorFunction<Field> & _HermitianRBSolver;
+    int CBfactorise;
+  public:
+
+    /////////////////////////////////////////////////////
+    // Wrap the usual normal equations Schur trick
+    /////////////////////////////////////////////////////
+  SchurRedBlackDiagTwoSolve(OperatorFunction<Field> &HermitianRBSolver)  :
+     _HermitianRBSolver(HermitianRBSolver) 
+    { 
+      CBfactorise=0;
+    };
+
+    template<class Matrix>
+      void operator() (Matrix & _Matrix,const Field &in, Field &out){
+
+      // FIXME CGdiagonalMee not implemented virtual function
+      // FIXME use CBfactorise to control schur decomp
+      GridBase *grid = _Matrix.RedBlackGrid();
+      GridBase *fgrid= _Matrix.Grid();
+
+      SchurDiagTwoOperator<Matrix,Field> _HermOpEO(_Matrix);
+ 
+      Field src_e(grid);
+      Field src_o(grid);
+      Field sol_e(grid);
+      Field sol_o(grid);
+      Field   tmp(grid);
+      Field  Mtmp(grid);
+      Field resid(fgrid);
+
+      pickCheckerboard(Even,src_e,in);
+      pickCheckerboard(Odd ,src_o,in);
+      pickCheckerboard(Even,sol_e,out);
+      pickCheckerboard(Odd ,sol_o,out);
+    
+      /////////////////////////////////////////////////////
+      // src_o = Mdag * (source_o - Moe MeeInv source_e)
+      /////////////////////////////////////////////////////
+      _Matrix.MooeeInv(src_e,tmp);     assert(  tmp.checkerboard ==Even);
+      _Matrix.Meooe   (tmp,Mtmp);      assert( Mtmp.checkerboard ==Odd);     
+      tmp=src_o-Mtmp;                  assert(  tmp.checkerboard ==Odd);     
+
+      // get the right MpcDag
+      _HermOpEO.MpcDag(tmp,src_o);     assert(src_o.checkerboard ==Odd);       
+
+      //////////////////////////////////////////////////////////////
+      // Call the red-black solver
+      //////////////////////////////////////////////////////////////
+      std::cout<<GridLogMessage << "SchurRedBlack solver calling the MpcDagMp solver" <<std::endl;
+//      _HermitianRBSolver(_HermOpEO,src_o,sol_o);  assert(sol_o.checkerboard==Odd);
+      _HermitianRBSolver(_HermOpEO,src_o,tmp);  assert(tmp.checkerboard==Odd);
+      _Matrix.MooeeInv(tmp,sol_o);        assert(  sol_o.checkerboard   ==Odd);
+
+      ///////////////////////////////////////////////////
+      // sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
+      ///////////////////////////////////////////////////
+      _Matrix.Meooe(sol_o,tmp);        assert(  tmp.checkerboard   ==Even);
+      src_e = src_e-tmp;               assert(  src_e.checkerboard ==Even);
+      _Matrix.MooeeInv(src_e,sol_e);   assert(  sol_e.checkerboard ==Even);
+     
+      setCheckerboard(out,sol_e); assert(  sol_e.checkerboard ==Even);
+      setCheckerboard(out,sol_o); assert(  sol_o.checkerboard ==Odd );
+
+      // Verify the unprec residual
+      _Matrix.M(out,resid); 
+      resid = resid-in;
+      RealD ns = norm2(in);
+      RealD nr = norm2(resid);
+
+      std::cout<<GridLogMessage << "SchurRedBlackDiagTwo solver true unprec resid "<< std::sqrt(nr/ns) <<" nr "<< nr <<" ns "<<ns << std::endl;
+    }     
+  };
+  ///////////////////////////////////////////////////////////////////////////////////////////////////////
+  // Take a matrix and form a Red Black solver calling a Herm solver
+  // Use of RB info prevents making SchurRedBlackSolve conform to standard interface
+  ///////////////////////////////////////////////////////////////////////////////////////////////////////
+  template<class Field> class SchurRedBlackDiagTwoMixed {
+  private:
+    LinearFunction<Field> & _HermitianRBSolver;
+    int CBfactorise;
+  public:
+
+    /////////////////////////////////////////////////////
+    // Wrap the usual normal equations Schur trick
+    /////////////////////////////////////////////////////
+  SchurRedBlackDiagTwoMixed(LinearFunction<Field> &HermitianRBSolver)  :
+     _HermitianRBSolver(HermitianRBSolver) 
+    { 
+      CBfactorise=0;
+    };
+
+    template<class Matrix>
+      void operator() (Matrix & _Matrix,const Field &in, Field &out){
+
+      // FIXME CGdiagonalMee not implemented virtual function
+      // FIXME use CBfactorise to control schur decomp
+      GridBase *grid = _Matrix.RedBlackGrid();
+      GridBase *fgrid= _Matrix.Grid();
+
+      SchurDiagTwoOperator<Matrix,Field> _HermOpEO(_Matrix);
+ 
+      Field src_e(grid);
+      Field src_o(grid);
+      Field sol_e(grid);
+      Field sol_o(grid);
+      Field   tmp(grid);
+      Field  Mtmp(grid);
+      Field resid(fgrid);
+
+      pickCheckerboard(Even,src_e,in);
+      pickCheckerboard(Odd ,src_o,in);
+      pickCheckerboard(Even,sol_e,out);
+      pickCheckerboard(Odd ,sol_o,out);
+    
+      /////////////////////////////////////////////////////
+      // src_o = Mdag * (source_o - Moe MeeInv source_e)
+      /////////////////////////////////////////////////////
+      _Matrix.MooeeInv(src_e,tmp);     assert(  tmp.checkerboard ==Even);
+      _Matrix.Meooe   (tmp,Mtmp);      assert( Mtmp.checkerboard ==Odd);     
+      tmp=src_o-Mtmp;                  assert(  tmp.checkerboard ==Odd);     
+
+      // get the right MpcDag
+      _HermOpEO.MpcDag(tmp,src_o);     assert(src_o.checkerboard ==Odd);       
+
+      //////////////////////////////////////////////////////////////
+      // Call the red-black solver
+      //////////////////////////////////////////////////////////////
+      std::cout<<GridLogMessage << "SchurRedBlack solver calling the MpcDagMp solver" <<std::endl;
+//      _HermitianRBSolver(_HermOpEO,src_o,sol_o);  assert(sol_o.checkerboard==Odd);
+//      _HermitianRBSolver(_HermOpEO,src_o,tmp);  assert(tmp.checkerboard==Odd);
+      _HermitianRBSolver(src_o,tmp);  assert(tmp.checkerboard==Odd);
+      _Matrix.MooeeInv(tmp,sol_o);        assert(  sol_o.checkerboard   ==Odd);
+
+      ///////////////////////////////////////////////////
+      // sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
+      ///////////////////////////////////////////////////
+      _Matrix.Meooe(sol_o,tmp);        assert(  tmp.checkerboard   ==Even);
+      src_e = src_e-tmp;               assert(  src_e.checkerboard ==Even);
+      _Matrix.MooeeInv(src_e,sol_e);   assert(  sol_e.checkerboard ==Even);
+     
+      setCheckerboard(out,sol_e); assert(  sol_e.checkerboard ==Even);
+      setCheckerboard(out,sol_o); assert(  sol_o.checkerboard ==Odd );
+
+      // Verify the unprec residual
+      _Matrix.M(out,resid); 
+      resid = resid-in;
+      RealD ns = norm2(in);
+      RealD nr = norm2(resid);
+
+      std::cout<<GridLogMessage << "SchurRedBlackDiagTwo solver true unprec resid "<< std::sqrt(nr/ns) <<" nr "<< nr <<" ns "<<ns << std::endl;
+    }     
+  };
+
 }
 #endif

From bb7378cfc3e059e7aaf53690137a4a5cb54ec8a7 Mon Sep 17 00:00:00 2001
From: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 10 Oct 2017 12:02:18 +0100
Subject: [PATCH 109/115] Schur for staggered

---
 lib/algorithms/LinearOperator.h              |  80 +-----------
 tests/solver/Test_staggered_block_cg_prec.cc | 130 +++++++++++++++++++
 tests/solver/Test_staggered_cg_prec.cc       |   6 +-
 3 files changed, 135 insertions(+), 81 deletions(-)
 create mode 100644 tests/solver/Test_staggered_block_cg_prec.cc

diff --git a/lib/algorithms/LinearOperator.h b/lib/algorithms/LinearOperator.h
index 6e4da248..d402c5b7 100644
--- a/lib/algorithms/LinearOperator.h
+++ b/lib/algorithms/LinearOperator.h
@@ -162,15 +162,10 @@ namespace Grid {
 	_Mat.M(in,out);
       }
       void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
-	ComplexD dot;
-
 	_Mat.M(in,out);
 	
-	dot= innerProduct(in,out);
-	n1=real(dot);
-
-	dot = innerProduct(out,out);
-	n2=real(dot);
+	ComplexD dot= innerProduct(in,out); n1=real(dot);
+	n2=norm2(out);
       }
       void HermOp(const Field &in, Field &out){
 	_Mat.M(in,out);
@@ -309,9 +304,9 @@ namespace Grid {
       SchurStaggeredOperator (Matrix &Mat): _Mat(Mat){};
       virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
 	ComplexD dot;
-	n2=Mpc(in,out);
+	n2 = Mpc(in,out);
 	dot= innerProduct(in,out);
-	n1= real(dot);
+	n1 = real(dot);
       }
       virtual void HermOp(const Field &in, Field &out){
 	Mpc(in,out);
@@ -333,73 +328,6 @@ namespace Grid {
     };
     template<class Matrix,class Field> using SchurStagOperator = SchurStaggeredOperator<Matrix,Field>;
 
-  // This is specific to (Z)mobius fermions
-  template<class Matrix, class Field>
-    class KappaSimilarityTransform {
-  public:
-
-    typedef typename Matrix::Coeff_t                     Coeff_t;
-    std::vector<Coeff_t> kappa, kappaDag, kappaInv, kappaInvDag;
-
-    KappaSimilarityTransform (Matrix &zmob) {
-      for (int i=0;i<(int)zmob.bs.size();i++) {
-	Coeff_t k = 1.0 / ( 2.0 * (zmob.bs[i] *(4 - zmob.M5) + 1.0) );
-	kappa.push_back( k );
-	kappaDag.push_back( conj(k) );
-	kappaInv.push_back( 1.0 / k );
-	kappaInvDag.push_back( 1.0 / conj(k) );
-      }
-    }
-
-  template<typename vobj>
-    void sscale(const Lattice<vobj>& in, Lattice<vobj>& out, Coeff_t* s) {
-    GridBase *grid=out._grid;
-    out.checkerboard = in.checkerboard;
-    assert(grid->_simd_layout[0] == 1); // should be fine for ZMobius for now
-    int Ls = grid->_rdimensions[0];
-    parallel_for(int ss=0;ss<grid->oSites();ss++){
-      vobj tmp = s[ss % Ls]*in._odata[ss];
-      vstream(out._odata[ss],tmp);
-    }
-  }
-
-  RealD sscale_norm(const Field& in, Field& out, Coeff_t* s) {
-    sscale(in,out,s);
-    return norm2(out);
-  }
-
-  virtual RealD M       (const Field& in, Field& out) { return sscale_norm(in,out,&kappa[0]);   }
-  virtual RealD MDag    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaDag[0]);}
-  virtual RealD MInv    (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInv[0]);}
-  virtual RealD MInvDag (const Field& in, Field& out) { return sscale_norm(in,out,&kappaInvDag[0]);}
-
-  };
-
-  template<class Matrix,class Field>
-    class SchurDiagTwoKappaOperator :  public SchurOperatorBase<Field> {
-  public:
-    KappaSimilarityTransform<Matrix, Field> _S;
-    SchurDiagTwoOperator<Matrix, Field> _Mat;
-
-    SchurDiagTwoKappaOperator (Matrix &Mat): _S(Mat), _Mat(Mat) {};
-
-    virtual  RealD Mpc      (const Field &in, Field &out) {
-      Field tmp(in._grid);
-
-      _S.MInv(in,out);
-      _Mat.Mpc(out,tmp);
-      return _S.M(tmp,out);
-
-    }
-    virtual  RealD MpcDag   (const Field &in, Field &out){
-      Field tmp(in._grid);
-
-      _S.MDag(in,out);
-      _Mat.MpcDag(out,tmp);
-      return _S.MInvDag(tmp,out);
-    }
-  };
-
 
     /////////////////////////////////////////////////////////////
     // Base classes for functions of operators
diff --git a/tests/solver/Test_staggered_block_cg_prec.cc b/tests/solver/Test_staggered_block_cg_prec.cc
new file mode 100644
index 00000000..1d0117e0
--- /dev/null
+++ b/tests/solver/Test_staggered_block_cg_prec.cc
@@ -0,0 +1,130 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./tests/Test_wilson_cg_unprec.cc
+
+    Copyright (C) 2015
+
+Author: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#include <Grid/Grid.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+template<class d>
+struct scal {
+  d internal;
+};
+
+  Gamma::Algebra Gmu [] = {
+    Gamma::Algebra::GammaX,
+    Gamma::Algebra::GammaY,
+    Gamma::Algebra::GammaZ,
+    Gamma::Algebra::GammaT
+  };
+
+int main (int argc, char ** argv)
+{
+  typedef typename ImprovedStaggeredFermion5DR::FermionField FermionField; 
+  typedef typename ImprovedStaggeredFermion5DR::ComplexField ComplexField; 
+  typename ImprovedStaggeredFermion5DR::ImplParams params; 
+
+  const int Ls=8;
+
+  Grid_init(&argc,&argv);
+
+  std::vector<int> latt_size   = GridDefaultLatt();
+  std::vector<int> simd_layout = GridDefaultSimd(Nd,vComplex::Nsimd());
+  std::vector<int> mpi_layout  = GridDefaultMpi();
+
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+
+  std::vector<int> seeds({1,2,3,4});
+  GridParallelRNG pRNG(UGrid );  pRNG.SeedFixedIntegers(seeds);
+  GridParallelRNG pRNG5(FGrid);  pRNG5.SeedFixedIntegers(seeds);
+
+  FermionField src(FGrid); random(pRNG5,src);
+  FermionField src_o(FrbGrid);   pickCheckerboard(Odd,src_o,src);
+  FermionField result_o(FrbGrid); result_o=zero; 
+  RealD nrm = norm2(src);
+
+  LatticeGaugeField Umu(UGrid); SU3::HotConfiguration(pRNG,Umu);
+
+  RealD mass=0.003;
+  ImprovedStaggeredFermion5DR Ds(Umu,Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass); 
+  SchurDiagMooeeOperator<ImprovedStaggeredFermion5DR,FermionField> HermOp(Ds);
+
+  ConjugateGradient<FermionField> CG(1.0e-8,10000);
+  int blockDim = 0;
+  BlockConjugateGradient<FermionField>    BCGrQ(BlockCGrQ,blockDim,1.0e-8,10000);
+  BlockConjugateGradient<FermionField>    BCG  (BlockCG,blockDim,1.0e-8,10000);
+  BlockConjugateGradient<FermionField>    mCG  (CGmultiRHS,blockDim,1.0e-8,10000);
+
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+  std::cout << GridLogMessage << " Calling 4d CG "<<std::endl;
+  std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
+  ImprovedStaggeredFermionR Ds4d(Umu,Umu,*UGrid,*UrbGrid,mass);
+  SchurDiagMooeeOperator<ImprovedStaggeredFermionR,FermionField> HermOp4d(Ds4d);
+  FermionField src4d(UGrid); random(pRNG,src4d);
+  FermionField src4d_o(UrbGrid);   pickCheckerboard(Odd,src4d_o,src4d);
+  FermionField result4d_o(UrbGrid); 
+
+  result4d_o=zero;
+  CG(HermOp4d,src4d_o,result4d_o);
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+
+
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+  std::cout << GridLogMessage << " Calling 5d CG for "<<Ls <<" right hand sides" <<std::endl;
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+  Ds.ZeroCounters();
+  result_o=zero;
+  CG(HermOp,src_o,result_o);
+  Ds.Report();
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+  std::cout << GridLogMessage << " Calling multiRHS CG for "<<Ls <<" right hand sides" <<std::endl;
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+  Ds.ZeroCounters();
+  result_o=zero;
+  mCG(HermOp,src_o,result_o);
+  Ds.Report();
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+  std::cout << GridLogMessage << " Calling Block CG for "<<Ls <<" right hand sides" <<std::endl;
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+  Ds.ZeroCounters();
+  result_o=zero;
+  BCGrQ(HermOp,src_o,result_o);
+  Ds.Report();
+  std::cout << GridLogMessage << "************************************************************************ "<<std::endl;
+
+
+  Grid_finalize();
+}
diff --git a/tests/solver/Test_staggered_cg_prec.cc b/tests/solver/Test_staggered_cg_prec.cc
index d771f22e..97251435 100644
--- a/tests/solver/Test_staggered_cg_prec.cc
+++ b/tests/solver/Test_staggered_cg_prec.cc
@@ -71,7 +71,7 @@ int main (int argc, char ** argv)
     volume=volume*latt_size[mu];
   }  
   
-  RealD mass=0.1;
+  RealD mass=0.003;
   ImprovedStaggeredFermionR Ds(Umu,Umu,Grid,RBGrid,mass);
 
   FermionField res_o(&RBGrid); 
@@ -88,9 +88,5 @@ int main (int argc, char ** argv)
   HermOpEO.Mpc(res_o,tmp);
   std::cout << "check Mpc resid " << axpy_norm(tmp,-1.0,src_o,tmp)/norm2(src_o) << "\n";
 
-  RealD n1,n2;
-  HermOpEO.MpcDagMpc(res_o,tmp,n1,n2);
-  std::cout << "check MpcDagMpc resid " << axpy_norm(tmp,-1.0,src_o,tmp)/norm2(src_o) << "\n";
-
   Grid_finalize();
 }

From 4eb8bbbebeae35b42810f064b521b7f43396d823 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 10 Oct 2017 13:48:51 +0100
Subject: [PATCH 110/115] Christop mods

---
 lib/algorithms/approx/Chebyshev.h | 42 +++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/lib/algorithms/approx/Chebyshev.h b/lib/algorithms/approx/Chebyshev.h
index 2793f138..f8c21a05 100644
--- a/lib/algorithms/approx/Chebyshev.h
+++ b/lib/algorithms/approx/Chebyshev.h
@@ -8,6 +8,7 @@
 
 Author: Peter Boyle <paboyle@ph.ed.ac.uk>
 Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Christoph Lehner <clehner@bnl.gov>
 
     This program is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -193,6 +194,47 @@ namespace Grid {
       return sum;
     };
 
+    RealD approxD(RealD x)
+    {
+      RealD Un;
+      RealD Unm;
+      RealD Unp;
+      
+      RealD y=( x-0.5*(hi+lo))/(0.5*(hi-lo));
+      
+      RealD U0=1;
+      RealD U1=2*y;
+      
+      RealD sum;
+      sum = Coeffs[1]*U0;
+      sum+= Coeffs[2]*U1*2.0;
+      
+      Un =U1;
+      Unm=U0;
+      for(int i=2;i<order-1;i++){
+	Unp=2*y*Un-Unm;
+	Unm=Un;
+	Un =Unp;
+	sum+= Un*Coeffs[i+1]*(i+1.0);
+      }
+      return sum/(0.5*(hi-lo));
+    };
+    
+    RealD approxInv(RealD z, RealD x0, int maxiter, RealD resid) {
+      RealD x = x0;
+      RealD eps;
+      
+      int i;
+      for (i=0;i<maxiter;i++) {
+	eps = approx(x) - z;
+	if (fabs(eps / z) < resid)
+	  return x;
+	x = x - eps / approxD(x);
+      }
+      
+      return std::numeric_limits<double>::quiet_NaN();
+    }
+    
     // Implement the required interface
     void operator() (LinearOperatorBase<Field> &Linop, const Field &in, Field &out) {
 

From a1d80282eca8df8c1c7eb521c48c3aa78ccdb389 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 10 Oct 2017 13:49:31 +0100
Subject: [PATCH 111/115] cb factorise

---
 lib/algorithms/iterative/SchurRedBlack.h | 27 ++++++++++++++++++------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/lib/algorithms/iterative/SchurRedBlack.h b/lib/algorithms/iterative/SchurRedBlack.h
index b6eab762..a309386b 100644
--- a/lib/algorithms/iterative/SchurRedBlack.h
+++ b/lib/algorithms/iterative/SchurRedBlack.h
@@ -53,13 +53,28 @@ Author: Peter Boyle <paboyle@ph.ed.ac.uk>
    *     M psi = eta
    ***********************
    *Odd
-   * i)   (D_oo)^{\dag} D_oo psi_o = (D_oo)^dag L^{-1}  eta_o
+   * i)                 D_oo psi_o =  L^{-1}  eta_o
    *                        eta_o' = (D_oo)^dag (eta_o - Moe Mee^{-1} eta_e)
+   *      (D_oo)^{\dag} D_oo psi_o = (D_oo)^dag L^{-1}  eta_o
    *Even
    * ii)  Mee psi_e + Meo psi_o = src_e
    *
    *   => sol_e = M_ee^-1 * ( src_e - Meo sol_o )...
    *
+   * 
+   * TODO: Other options:
+   * 
+   * a) change checkerboards for Schur e<->o
+   *
+   * Left precon by Moo^-1
+   * b) Doo^{dag} M_oo^-dag Moo^-1 Doo psi_0 =  (D_oo)^dag M_oo^-dag Moo^-1 L^{-1}  eta_o
+   *                              eta_o'     = (D_oo)^dag  M_oo^-dag Moo^-1 (eta_o - Moe Mee^{-1} eta_e)
+   *
+   * Right precon by Moo^-1
+   * c) M_oo^-dag Doo^{dag} Doo Moo^-1 phi_0 = M_oo^-dag (D_oo)^dag L^{-1}  eta_o
+   *                              eta_o'     = M_oo^-dag (D_oo)^dag (eta_o - Moe Mee^{-1} eta_e)
+   *                              psi_o = M_oo^-1 phi_o
+   * TODO: Deflation 
    */
 namespace Grid {
 
@@ -155,12 +170,10 @@ namespace Grid {
     /////////////////////////////////////////////////////
     // Wrap the usual normal equations Schur trick
     /////////////////////////////////////////////////////
-  SchurRedBlackDiagMooeeSolve(OperatorFunction<Field> &HermitianRBSolver)  :
-     _HermitianRBSolver(HermitianRBSolver) 
-    { 
-      CBfactorise=0;
-    };
-
+  SchurRedBlackDiagMooeeSolve(OperatorFunction<Field> &HermitianRBSolver,int cb=0)  :  _HermitianRBSolver(HermitianRBSolver) 
+  { 
+    CBfactorise=cb;
+  };
     template<class Matrix>
       void operator() (Matrix & _Matrix,const Field &in, Field &out){
 

From 1374c943d4cbb493a6a909a54b7c55471b677a32 Mon Sep 17 00:00:00 2001
From: Azusa Yamaguchi <ayamaguc@staffmail.ed.ac.uk>
Date: Tue, 10 Oct 2017 13:59:50 +0100
Subject: [PATCH 112/115] Correct Schur operator called

---
 tests/solver/Test_staggered_block_cg_prec.cc | 4 ++--
 tests/solver/Test_staggered_cg_prec.cc       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/solver/Test_staggered_block_cg_prec.cc b/tests/solver/Test_staggered_block_cg_prec.cc
index 1d0117e0..0076e5a0 100644
--- a/tests/solver/Test_staggered_block_cg_prec.cc
+++ b/tests/solver/Test_staggered_block_cg_prec.cc
@@ -76,7 +76,7 @@ int main (int argc, char ** argv)
 
   RealD mass=0.003;
   ImprovedStaggeredFermion5DR Ds(Umu,Umu,*FGrid,*FrbGrid,*UGrid,*UrbGrid,mass); 
-  SchurDiagMooeeOperator<ImprovedStaggeredFermion5DR,FermionField> HermOp(Ds);
+  SchurStaggeredOperator<ImprovedStaggeredFermion5DR,FermionField> HermOp(Ds);
 
   ConjugateGradient<FermionField> CG(1.0e-8,10000);
   int blockDim = 0;
@@ -88,7 +88,7 @@ int main (int argc, char ** argv)
   std::cout << GridLogMessage << " Calling 4d CG "<<std::endl;
   std::cout << GridLogMessage << "****************************************************************** "<<std::endl;
   ImprovedStaggeredFermionR Ds4d(Umu,Umu,*UGrid,*UrbGrid,mass);
-  SchurDiagMooeeOperator<ImprovedStaggeredFermionR,FermionField> HermOp4d(Ds4d);
+  SchurStaggeredOperator<ImprovedStaggeredFermionR,FermionField> HermOp4d(Ds4d);
   FermionField src4d(UGrid); random(pRNG,src4d);
   FermionField src4d_o(UrbGrid);   pickCheckerboard(Odd,src4d_o,src4d);
   FermionField result4d_o(UrbGrid); 
diff --git a/tests/solver/Test_staggered_cg_prec.cc b/tests/solver/Test_staggered_cg_prec.cc
index 97251435..9a458f1f 100644
--- a/tests/solver/Test_staggered_cg_prec.cc
+++ b/tests/solver/Test_staggered_cg_prec.cc
@@ -79,7 +79,7 @@ int main (int argc, char ** argv)
   pickCheckerboard(Odd,src_o,src);
   res_o=zero;
 
-  SchurDiagMooeeOperator<ImprovedStaggeredFermionR,FermionField> HermOpEO(Ds);
+  SchurStaggeredOperator<ImprovedStaggeredFermionR,FermionField> HermOpEO(Ds);
   ConjugateGradient<FermionField> CG(1.0e-8,10000);
   CG(HermOpEO,src_o,res_o);
 

From bf58557fb1ec710c766e19c9a8809b0a352de239 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Tue, 10 Oct 2017 14:15:11 +0100
Subject: [PATCH 113/115] Block compressed Lanczos

---
 lib/algorithms/LinearOperator.h               |   16 +-
 .../BlockImplicitlyRestartedLanczos.h         |  754 ++++++++++++
 .../BlockProjector.h                          |  143 +++
 .../BlockedGrid.h                             |  401 ++++++
 .../FieldBasisVector.h                        |  163 +++
 .../FieldVectorIO.h                           | 1085 +++++++++++++++++
 .../action/fermion/DomainWallEOFAFermion.cc   |   12 +-
 lib/qcd/action/fermion/MobiusEOFAFermion.cc   |   14 +-
 tests/solver/Params.h                         |  136 +++
 tests/solver/Test_dwf_compressed_lanczos.cc   |  727 +++++++++++
 10 files changed, 3432 insertions(+), 19 deletions(-)
 create mode 100644 lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h
 create mode 100644 lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockProjector.h
 create mode 100644 lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockedGrid.h
 create mode 100644 lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldBasisVector.h
 create mode 100644 lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldVectorIO.h
 create mode 100644 tests/solver/Params.h
 create mode 100644 tests/solver/Test_dwf_compressed_lanczos.cc

diff --git a/lib/algorithms/LinearOperator.h b/lib/algorithms/LinearOperator.h
index d402c5b7..f1b8820e 100644
--- a/lib/algorithms/LinearOperator.h
+++ b/lib/algorithms/LinearOperator.h
@@ -207,7 +207,6 @@ namespace Grid {
       void OpDir  (const Field &in, Field &out,int dir,int disp) {
 	assert(0);
       }
-
     };
     template<class Matrix,class Field>
       class SchurDiagMooeeOperator :  public SchurOperatorBase<Field> {
@@ -265,7 +264,6 @@ namespace Grid {
 	return axpy_norm(out,-1.0,tmp,in);
       }
     };
-
     template<class Matrix,class Field>
       class SchurDiagTwoOperator :  public SchurOperatorBase<Field> {
     protected:
@@ -294,8 +292,15 @@ namespace Grid {
 	return axpy_norm(out,-1.0,tmp,in);
       }
     };
-
-      //
+    ///////////////////////////////////////////////////////////////////////////////////////////////////
+    // Left  handed Moo^-1 ; (Moo - Moe Mee^-1 Meo) psi = eta  -->  ( 1 - Moo^-1 Moe Mee^-1 Meo ) psi = Moo^-1 eta
+    // Right handed Moo^-1 ; (Moo - Moe Mee^-1 Meo) Moo^-1 Moo psi = eta  -->  ( 1 - Moe Mee^-1 Meo ) Moo^-1 phi=eta ; psi = Moo^-1 phi
+    ///////////////////////////////////////////////////////////////////////////////////////////////////
+    template<class Matrix,class Field> using SchurDiagOneRH = SchurDiagTwoOperator<Matrix,Field> ;
+    template<class Matrix,class Field> using SchurDiagOneLH = SchurDiagOneOperator<Matrix,Field> ;
+    ///////////////////////////////////////////////////////////////////////////////////////////////////
+    //  Staggered use
+    ///////////////////////////////////////////////////////////////////////////////////////////////////
     template<class Matrix,class Field>
       class SchurStaggeredOperator :  public SchurOperatorBase<Field> {
     protected:
@@ -303,9 +308,8 @@ namespace Grid {
     public:
       SchurStaggeredOperator (Matrix &Mat): _Mat(Mat){};
       virtual void HermOpAndNorm(const Field &in, Field &out,RealD &n1,RealD &n2){
-	ComplexD dot;
 	n2 = Mpc(in,out);
-	dot= innerProduct(in,out);
+	ComplexD dot= innerProduct(in,out);
 	n1 = real(dot);
       }
       virtual void HermOp(const Field &in, Field &out){
diff --git a/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h
new file mode 100644
index 00000000..82a00efa
--- /dev/null
+++ b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h
@@ -0,0 +1,754 @@
+    /*************************************************************************************
+
+    Grid physics library, www.github.com/paboyle/Grid 
+
+    Source file: ./lib/algorithms/iterative/ImplicitlyRestartedLanczos.h
+
+    Copyright (C) 2015
+
+Author: Peter Boyle <paboyle@ph.ed.ac.uk>
+Author: paboyle <paboyle@ph.ed.ac.uk>
+Author: Chulwoo Jung <chulwoo@bnl.gov>
+Author: Christoph Lehner <clehner@bnl.gov>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+    See the full license in the file "LICENSE" in the top level distribution directory
+    *************************************************************************************/
+    /*  END LEGAL */
+#ifndef GRID_BIRL_H
+#define GRID_BIRL_H
+
+#include <string.h> //memset
+
+#include <zlib.h>
+#include <sys/stat.h>
+
+#include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockedGrid.h>
+#include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldBasisVector.h>
+#include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockProjector.h>
+#include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldVectorIO.h>
+
+namespace Grid { 
+
+/////////////////////////////////////////////////////////////
+// Implicitly restarted lanczos
+/////////////////////////////////////////////////////////////
+
+ template<class Field> 
+ class BlockImplicitlyRestartedLanczos {
+
+    const RealD small = 1.0e-16;
+public:       
+    int lock;
+    int get;
+    int Niter;
+    int converged;
+
+    int Nminres; // Minimum number of restarts; only check for convergence after
+    int Nstop;   // Number of evecs checked for convergence
+    int Nk;      // Number of converged sought
+    int Np;      // Np -- Number of spare vecs in kryloc space
+    int Nm;      // Nm -- total number of vectors
+
+    int orth_period;
+
+    RealD OrthoTime;
+
+    RealD eresid, betastp;
+    SortEigen<Field> _sort;
+    LinearFunction<Field> &_HermOp;
+    LinearFunction<Field> &_HermOpTest;
+    /////////////////////////
+    // Constructor
+    /////////////////////////
+
+    BlockImplicitlyRestartedLanczos(
+			       LinearFunction<Field> & HermOp,
+			       LinearFunction<Field> & HermOpTest,
+			       int _Nstop, // sought vecs
+			       int _Nk, // sought vecs
+			       int _Nm, // spare vecs
+			       RealD _eresid, // resid in lmdue deficit 
+			       RealD _betastp, // if beta(k) < betastp: converged
+			       int _Niter, // Max iterations
+			       int _Nminres, int _orth_period = 1) :
+      _HermOp(HermOp),
+      _HermOpTest(HermOpTest),
+      Nstop(_Nstop),
+      Nk(_Nk),
+      Nm(_Nm),
+      eresid(_eresid),
+      betastp(_betastp),
+      Niter(_Niter),
+	Nminres(_Nminres),
+	orth_period(_orth_period)
+    { 
+      Np = Nm-Nk; assert(Np>0);
+    };
+
+    BlockImplicitlyRestartedLanczos(
+			       LinearFunction<Field> & HermOp,
+			       LinearFunction<Field> & HermOpTest,
+			       int _Nk, // sought vecs
+			       int _Nm, // spare vecs
+			       RealD _eresid, // resid in lmdue deficit 
+			       RealD _betastp, // if beta(k) < betastp: converged
+			       int _Niter, // Max iterations
+			       int _Nminres,
+			       int _orth_period = 1) : 
+      _HermOp(HermOp),
+      _HermOpTest(HermOpTest),
+      Nstop(_Nk),
+      Nk(_Nk),
+      Nm(_Nm),
+      eresid(_eresid),
+      betastp(_betastp),
+      Niter(_Niter),
+	Nminres(_Nminres),
+	orth_period(_orth_period)
+    { 
+      Np = Nm-Nk; assert(Np>0);
+    };
+
+
+/* Saad PP. 195
+1. Choose an initial vector v1 of 2-norm unity. Set β1 ≡ 0, v0 ≡ 0
+2. For k = 1,2,...,m Do:
+3. wk:=Avk−βkv_{k−1}      
+4. αk:=(wk,vk)       // 
+5. wk:=wk−αkvk       // wk orthog vk 
+6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop
+7. vk+1 := wk/βk+1
+8. EndDo
+ */
+    void step(std::vector<RealD>& lmd,
+	      std::vector<RealD>& lme, 
+	      BasisFieldVector<Field>& evec,
+	      Field& w,int Nm,int k)
+    {
+      assert( k< Nm );
+
+      GridStopWatch gsw_op,gsw_o;
+
+      Field& evec_k = evec[k];
+
+      gsw_op.Start();
+      _HermOp(evec_k,w);
+      gsw_op.Stop();
+
+      if(k>0){
+	w -= lme[k-1] * evec[k-1];
+      }    
+
+      ComplexD zalph = innerProduct(evec_k,w); // 4. αk:=(wk,vk)
+      RealD     alph = real(zalph);
+
+      w = w - alph * evec_k;// 5. wk:=wk−αkvk
+
+      RealD beta = normalise(w); // 6. βk+1 := ∥wk∥2. If βk+1 = 0 then Stop
+                                 // 7. vk+1 := wk/βk+1
+
+      std::cout<<GridLogMessage << "alpha[" << k << "] = " << zalph << " beta[" << k << "] = "<<beta<<std::endl;
+      const RealD tiny = 1.0e-20;
+      if ( beta < tiny ) { 
+	std::cout<<GridLogMessage << " beta is tiny "<<beta<<std::endl;
+     }
+      lmd[k] = alph;
+      lme[k]  = beta;
+
+      gsw_o.Start();
+      if (k>0 && k % orth_period == 0) { 
+	orthogonalize(w,evec,k); // orthonormalise
+      }
+      gsw_o.Stop();
+
+      if(k < Nm-1) { 
+	evec[k+1] = w;
+      }
+
+      std::cout << GridLogMessage << "Timing: operator=" << gsw_op.Elapsed() <<
+	" orth=" << gsw_o.Elapsed() << std::endl;
+
+    }
+
+    void qr_decomp(std::vector<RealD>& lmd,
+		   std::vector<RealD>& lme,
+		   int Nk,
+		   int Nm,
+		   std::vector<RealD>& Qt,
+		   RealD Dsh, 
+		   int kmin,
+		   int kmax)
+    {
+      int k = kmin-1;
+      RealD x;
+
+      RealD Fden = 1.0/hypot(lmd[k]-Dsh,lme[k]);
+      RealD c = ( lmd[k] -Dsh) *Fden;
+      RealD s = -lme[k] *Fden;
+      
+      RealD tmpa1 = lmd[k];
+      RealD tmpa2 = lmd[k+1];
+      RealD tmpb  = lme[k];
+
+      lmd[k]   = c*c*tmpa1 +s*s*tmpa2 -2.0*c*s*tmpb;
+      lmd[k+1] = s*s*tmpa1 +c*c*tmpa2 +2.0*c*s*tmpb;
+      lme[k]   = c*s*(tmpa1-tmpa2) +(c*c-s*s)*tmpb;
+      x        =-s*lme[k+1];
+      lme[k+1] = c*lme[k+1];
+      
+      for(int i=0; i<Nk; ++i){
+	RealD Qtmp1 = Qt[i+Nm*k  ];
+	RealD Qtmp2 = Qt[i+Nm*(k+1)];
+	Qt[i+Nm*k    ] = c*Qtmp1 - s*Qtmp2;
+	Qt[i+Nm*(k+1)] = s*Qtmp1 + c*Qtmp2; 
+      }
+
+      // Givens transformations
+      for(int k = kmin; k < kmax-1; ++k){
+
+	RealD Fden = 1.0/hypot(x,lme[k-1]);
+	RealD c = lme[k-1]*Fden;
+	RealD s = - x*Fden;
+	
+	RealD tmpa1 = lmd[k];
+	RealD tmpa2 = lmd[k+1];
+	RealD tmpb  = lme[k];
+
+	lmd[k]   = c*c*tmpa1 +s*s*tmpa2 -2.0*c*s*tmpb;
+	lmd[k+1] = s*s*tmpa1 +c*c*tmpa2 +2.0*c*s*tmpb;
+	lme[k]   = c*s*(tmpa1-tmpa2) +(c*c-s*s)*tmpb;
+	lme[k-1] = c*lme[k-1] -s*x;
+
+	if(k != kmax-2){
+	  x = -s*lme[k+1];
+	  lme[k+1] = c*lme[k+1];
+	}
+
+	for(int i=0; i<Nk; ++i){
+	  RealD Qtmp1 = Qt[i+Nm*k    ];
+	  RealD Qtmp2 = Qt[i+Nm*(k+1)];
+	  Qt[i+Nm*k    ] = c*Qtmp1 -s*Qtmp2;
+	  Qt[i+Nm*(k+1)] = s*Qtmp1 +c*Qtmp2;
+	}
+      }
+    }
+
+#ifdef USE_LAPACK_IRL
+#define LAPACK_INT int
+//long long
+    void diagonalize_lapack(std::vector<RealD>& lmd,
+			    std::vector<RealD>& lme, 
+			    int N1,
+			    int N2,
+			    std::vector<RealD>& Qt,
+			    GridBase *grid){
+
+      std::cout << GridLogMessage << "diagonalize_lapack start\n";
+      GridStopWatch gsw;
+
+      const int size = Nm;
+      //  tevals.resize(size);
+      //  tevecs.resize(size);
+      LAPACK_INT NN = N1;
+      std::vector<double> evals_tmp(NN);
+      std::vector<double> evec_tmp(NN*NN);
+      memset(&evec_tmp[0],0,sizeof(double)*NN*NN);
+      //  double AA[NN][NN];
+      std::vector<double> DD(NN);
+      std::vector<double> EE(NN);
+      for (int i = 0; i< NN; i++)
+	for (int j = i - 1; j <= i + 1; j++)
+	  if ( j < NN && j >= 0 ) {
+	    if (i==j) DD[i] = lmd[i];
+	    if (i==j) evals_tmp[i] = lmd[i];
+	    if (j==(i-1)) EE[j] = lme[j];
+	  }
+      LAPACK_INT evals_found;
+      LAPACK_INT lwork = ( (18*NN) > (1+4*NN+NN*NN)? (18*NN):(1+4*NN+NN*NN)) ;
+      LAPACK_INT liwork =  3+NN*10 ;
+      std::vector<LAPACK_INT> iwork(liwork);
+      std::vector<double> work(lwork);
+      std::vector<LAPACK_INT> isuppz(2*NN);
+      char jobz = 'V'; // calculate evals & evecs
+      char range = 'I'; // calculate all evals
+      //    char range = 'A'; // calculate all evals
+      char uplo = 'U'; // refer to upper half of original matrix
+      char compz = 'I'; // Compute eigenvectors of tridiagonal matrix
+      std::vector<int> ifail(NN);
+      LAPACK_INT info;
+      //  int total = QMP_get_number_of_nodes();
+      //  int node = QMP_get_node_number();
+      //  GridBase *grid = evec[0]._grid;
+      int total = grid->_Nprocessors;
+      int node = grid->_processor;
+      int interval = (NN/total)+1;
+      double vl = 0.0, vu = 0.0;
+      LAPACK_INT il = interval*node+1 , iu = interval*(node+1);
+      if (iu > NN)  iu=NN;
+      double tol = 0.0;
+      if (1) {
+	memset(&evals_tmp[0],0,sizeof(double)*NN);
+	if ( il <= NN){
+	  std::cout << GridLogMessage << "dstegr started" << std::endl; 
+	  gsw.Start();
+	  dstegr(&jobz, &range, &NN,
+		 (double*)&DD[0], (double*)&EE[0],
+		 &vl, &vu, &il, &iu, // these four are ignored if second parameteris 'A'
+		 &tol, // tolerance
+		 &evals_found, &evals_tmp[0], (double*)&evec_tmp[0], &NN,
+		 &isuppz[0],
+		 &work[0], &lwork, &iwork[0], &liwork,
+		 &info);
+	  gsw.Stop();
+	  std::cout << GridLogMessage << "dstegr completed in " << gsw.Elapsed() << std::endl;
+	  for (int i = iu-1; i>= il-1; i--){
+	    evals_tmp[i] = evals_tmp[i - (il-1)];
+	    if (il>1) evals_tmp[i-(il-1)]=0.;
+	    for (int j = 0; j< NN; j++){
+	      evec_tmp[i*NN + j] = evec_tmp[(i - (il-1)) * NN + j];
+	      if (il>1) evec_tmp[(i-(il-1)) * NN + j]=0.;
+	    }
+	  }
+	}
+	{
+	  //        QMP_sum_double_array(evals_tmp,NN);
+	  //        QMP_sum_double_array((double *)evec_tmp,NN*NN);
+	  grid->GlobalSumVector(&evals_tmp[0],NN);
+	  grid->GlobalSumVector(&evec_tmp[0],NN*NN);
+	}
+      } 
+      // cheating a bit. It is better to sort instead of just reversing it, but the document of the routine says evals are sorted in increasing order. qr gives evals in decreasing order.
+      for(int i=0;i<NN;i++){
+	for(int j=0;j<NN;j++)
+	  Qt[(NN-1-i)*N2+j]=evec_tmp[i*NN + j];
+	lmd [NN-1-i]=evals_tmp[i];
+      }
+
+      std::cout << GridLogMessage << "diagonalize_lapack complete\n";
+    }
+#undef LAPACK_INT 
+#endif
+
+
+    void diagonalize(std::vector<RealD>& lmd,
+		     std::vector<RealD>& lme, 
+		     int N2,
+		     int N1,
+		     std::vector<RealD>& Qt,
+		     GridBase *grid)
+    {
+
+#ifdef USE_LAPACK_IRL
+    const int check_lapack=0; // just use lapack if 0, check against lapack if 1
+
+    if(!check_lapack)
+	return diagonalize_lapack(lmd,lme,N2,N1,Qt,grid);
+
+	std::vector <RealD> lmd2(N1);
+	std::vector <RealD> lme2(N1);
+	std::vector<RealD> Qt2(N1*N1);
+         for(int k=0; k<N1; ++k){
+	    lmd2[k] = lmd[k];
+	    lme2[k] = lme[k];
+	  }
+         for(int k=0; k<N1*N1; ++k)
+	Qt2[k] = Qt[k];
+
+//	diagonalize_lapack(lmd2,lme2,Nm2,Nm,Qt,grid);
+#endif
+
+      int Niter = 10000*N1;
+      int kmin = 1;
+      int kmax = N2;
+      // (this should be more sophisticated)
+
+      for(int iter=0; ; ++iter){
+      if ( (iter+1)%(100*N1)==0) 
+      std::cout<<GridLogMessage << "[QL method] Not converged - iteration "<<iter+1<<"\n";
+
+	// determination of 2x2 leading submatrix
+	RealD dsub = lmd[kmax-1]-lmd[kmax-2];
+	RealD dd = sqrt(dsub*dsub + 4.0*lme[kmax-2]*lme[kmax-2]);
+	RealD Dsh = 0.5*(lmd[kmax-2]+lmd[kmax-1] +dd*(dsub/fabs(dsub)));
+	// (Dsh: shift)
+	
+	// transformation
+	qr_decomp(lmd,lme,N2,N1,Qt,Dsh,kmin,kmax);
+	
+	// Convergence criterion (redef of kmin and kamx)
+	for(int j=kmax-1; j>= kmin; --j){
+	  RealD dds = fabs(lmd[j-1])+fabs(lmd[j]);
+	  if(fabs(lme[j-1])+dds > dds){
+	    kmax = j+1;
+	    goto continued;
+	  }
+	}
+	Niter = iter;
+#ifdef USE_LAPACK_IRL
+    if(check_lapack){
+	const double SMALL=1e-8;
+	diagonalize_lapack(lmd2,lme2,N2,N1,Qt2,grid);
+	std::vector <RealD> lmd3(N2);
+         for(int k=0; k<N2; ++k) lmd3[k]=lmd[k];
+        _sort.push(lmd3,N2);
+        _sort.push(lmd2,N2);
+         for(int k=0; k<N2; ++k){
+	    if (fabs(lmd2[k] - lmd3[k]) >SMALL)  std::cout<<GridLogMessage <<"lmd(qr) lmd(lapack) "<< k << ": " << lmd2[k] <<" "<< lmd3[k] <<std::endl;
+//	    if (fabs(lme2[k] - lme[k]) >SMALL)  std::cout<<GridLogMessage <<"lme(qr)-lme(lapack) "<< k << ": " << lme2[k] - lme[k] <<std::endl;
+	  }
+         for(int k=0; k<N1*N1; ++k){
+//	    if (fabs(Qt2[k] - Qt[k]) >SMALL)  std::cout<<GridLogMessage <<"Qt(qr)-Qt(lapack) "<< k << ": " << Qt2[k] - Qt[k] <<std::endl;
+	}
+    }
+#endif
+	return;
+
+      continued:
+	for(int j=0; j<kmax-1; ++j){
+	  RealD dds = fabs(lmd[j])+fabs(lmd[j+1]);
+	  if(fabs(lme[j])+dds > dds){
+	    kmin = j+1;
+	    break;
+	  }
+	}
+      }
+      std::cout<<GridLogMessage << "[QL method] Error - Too many iteration: "<<Niter<<"\n";
+      abort();
+    }
+
+#if 1
+    template<typename T>
+    static RealD normalise(T& v) 
+    {
+      RealD nn = norm2(v);
+      nn = sqrt(nn);
+      v = v * (1.0/nn);
+      return nn;
+    }
+
+    void orthogonalize(Field& w,
+		       BasisFieldVector<Field>& evec,
+		       int k)
+    {
+      double t0=-usecond()/1e6;
+
+      evec.orthogonalize(w,k);
+
+      normalise(w);
+      t0+=usecond()/1e6;
+      OrthoTime +=t0;
+    }
+
+    void setUnit_Qt(int Nm, std::vector<RealD> &Qt) {
+      for(int i=0; i<Qt.size(); ++i) Qt[i] = 0.0;
+      for(int k=0; k<Nm; ++k) Qt[k + k*Nm] = 1.0;
+    }
+
+/* Rudy Arthur's thesis pp.137
+------------------------
+Require: M > K P = M − K †
+Compute the factorization AVM = VM HM + fM eM 
+repeat
+  Q=I
+  for i = 1,...,P do
+    QiRi =HM −θiI Q = QQi
+    H M = Q †i H M Q i
+  end for
+  βK =HM(K+1,K) σK =Q(M,K)
+  r=vK+1βK +rσK
+  VK =VM(1:M)Q(1:M,1:K)
+  HK =HM(1:K,1:K)
+  →AVK =VKHK +fKe†K † Extend to an M = K + P step factorization AVM = VMHM + fMeM
+until convergence
+*/
+
+    void calc(std::vector<RealD>& eval,
+	      BasisFieldVector<Field>& evec,
+	      const Field& src,
+	      int& Nconv,
+	      bool reverse,
+	      int SkipTest)
+      {
+
+	GridBase *grid = evec._v[0]._grid;//evec.get(0 + evec_offset)._grid;
+	assert(grid == src._grid);
+
+	std::cout<<GridLogMessage << " -- Nk = " << Nk << " Np = "<< Np << std::endl;
+	std::cout<<GridLogMessage << " -- Nm = " << Nm << std::endl;
+	std::cout<<GridLogMessage << " -- size of eval   = " << eval.size() << std::endl;
+	std::cout<<GridLogMessage << " -- size of evec  = " << evec.size() << std::endl;
+	
+	assert(Nm <= evec.size() && Nm <= eval.size());
+
+	// quickly get an idea of the largest eigenvalue to more properly normalize the residuum
+	RealD evalMaxApprox = 0.0;
+	{
+	  auto src_n = src;
+	  auto tmp = src;
+	  const int _MAX_ITER_IRL_MEVAPP_ = 50;
+	  for (int i=0;i<_MAX_ITER_IRL_MEVAPP_;i++) {
+	    _HermOpTest(src_n,tmp);
+	    RealD vnum = real(innerProduct(src_n,tmp)); // HermOp.
+	    RealD vden = norm2(src_n);
+	    RealD na = vnum/vden;
+	    if (fabs(evalMaxApprox/na - 1.0) < 0.05)
+	      i=_MAX_ITER_IRL_MEVAPP_;
+	    evalMaxApprox = na;
+	    std::cout << GridLogMessage << " Approximation of largest eigenvalue: " << evalMaxApprox << std::endl;
+	    src_n = tmp;
+	  }
+	}
+	
+	std::vector<RealD> lme(Nm);  
+	std::vector<RealD> lme2(Nm);
+	std::vector<RealD> eval2(Nm);
+	std::vector<RealD> eval2_copy(Nm);
+	std::vector<RealD> Qt(Nm*Nm);
+
+
+	Field f(grid);
+	Field v(grid);
+  
+	int k1 = 1;
+	int k2 = Nk;
+
+	Nconv = 0;
+
+	RealD beta_k;
+  
+	// Set initial vector
+	evec[0] = src;
+	normalise(evec[0]);
+	std:: cout<<GridLogMessage <<"norm2(evec[0])= " << norm2(evec[0])<<std::endl;
+	
+	// Initial Nk steps
+	OrthoTime=0.;
+	double t0=usecond()/1e6;
+	for(int k=0; k<Nk; ++k) step(eval,lme,evec,f,Nm,k);
+	double t1=usecond()/1e6;
+	std::cout<<GridLogMessage <<"IRL::Initial steps: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	std::cout<<GridLogMessage <<"IRL::Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
+	t1=usecond()/1e6;
+
+	// Restarting loop begins
+	for(int iter = 0; iter<Niter; ++iter){
+	  
+	  std::cout<<GridLogMessage<<"\n Restart iteration = "<< iter << std::endl;
+	  
+	  // 
+	  // Rudy does a sort first which looks very different. Getting fed up with sorting out the algo defs.
+	  // We loop over 
+	  //
+	  OrthoTime=0.;
+	  for(int k=Nk; k<Nm; ++k) step(eval,lme,evec,f,Nm,k);
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL:: "<<Np <<" steps: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  std::cout<<GridLogMessage <<"IRL::Initial steps:OrthoTime "<<OrthoTime<< "seconds"<<std::endl;
+	  f *= lme[Nm-1];
+	  
+	  t1=usecond()/1e6;
+
+	  
+	  // getting eigenvalues
+	  for(int k=0; k<Nm; ++k){
+	    eval2[k] = eval[k+k1-1];
+	    lme2[k] = lme[k+k1-1];
+	  }
+	  setUnit_Qt(Nm,Qt);
+	  diagonalize(eval2,lme2,Nm,Nm,Qt,grid);
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL:: diagonalize: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  
+	  // sorting
+	  eval2_copy = eval2;
+
+	  _sort.push(eval2,Nm);
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL:: eval sorting: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  
+	  // Implicitly shifted QR transformations
+	  setUnit_Qt(Nm,Qt);
+	  for(int ip=0; ip<k2; ++ip){
+	    std::cout<<GridLogMessage << "eval "<< ip << " "<< eval2[ip] << std::endl;
+	  }
+
+	  for(int ip=k2; ip<Nm; ++ip){ 
+	    std::cout<<GridLogMessage << "qr_decomp "<< ip << " "<< eval2[ip] << std::endl;
+	    qr_decomp(eval,lme,Nm,Nm,Qt,eval2[ip],k1,Nm);
+	    
+	  }
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL::qr_decomp: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  assert(k2<Nm);
+	  
+
+	  assert(k2<Nm);
+	  assert(k1>0);
+	  evec.rotate(Qt,k1-1,k2+1,0,Nm,Nm);
+	  
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL::QR rotation: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  fflush(stdout);
+	  
+	  // Compressed vector f and beta(k2)
+	  f *= Qt[Nm-1+Nm*(k2-1)];
+	  f += lme[k2-1] * evec[k2];
+	  beta_k = norm2(f);
+	  beta_k = sqrt(beta_k);
+	  std::cout<<GridLogMessage<<" beta(k) = "<<beta_k<<std::endl;
+	  
+	  RealD betar = 1.0/beta_k;
+	  evec[k2] = betar * f;
+	  lme[k2-1] = beta_k;
+	  
+	  // Convergence test
+	  for(int k=0; k<Nm; ++k){    
+	    eval2[k] = eval[k];
+	    lme2[k] = lme[k];
+
+	    std::cout<<GridLogMessage << "eval2[" << k << "] = " << eval2[k] << std::endl;
+	  }
+	  setUnit_Qt(Nm,Qt);
+	  diagonalize(eval2,lme2,Nk,Nm,Qt,grid);
+	  t1=usecond()/1e6;
+	  std::cout<<GridLogMessage <<"IRL::diagonalize: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	  
+	  
+	  Nconv = 0;
+	  
+	  if (iter >= Nminres) {
+	    std::cout << GridLogMessage << "Rotation to test convergence " << std::endl;
+	    
+	    Field ev0_orig(grid);
+	    ev0_orig = evec[0];
+	    
+	    evec.rotate(Qt,0,Nk,0,Nk,Nm);
+	    
+	    {
+	      std::cout << GridLogMessage << "Test convergence" << std::endl;
+	      Field B(grid);
+	      
+	      for(int j = 0; j<Nk; j+=SkipTest){
+		B=evec[j];
+		//std::cout << "Checkerboard: " << evec[j].checkerboard << std::endl; 
+		B.checkerboard = evec[0].checkerboard;
+
+		_HermOpTest(B,v);
+		
+		RealD vnum = real(innerProduct(B,v)); // HermOp.
+		RealD vden = norm2(B);
+		RealD vv0 = norm2(v);
+		eval2[j] = vnum/vden;
+		v -= eval2[j]*B;
+		RealD vv = norm2(v) / ::pow(evalMaxApprox,2.0);
+		std::cout.precision(13);
+		std::cout<<GridLogMessage << "[" << std::setw(3)<< std::setiosflags(std::ios_base::right) <<j<<"] "
+			 <<"eval = "<<std::setw(25)<< std::setiosflags(std::ios_base::left)<< eval2[j] << " (" << eval2_copy[j] << ")"
+			 <<" |H B[i] - eval[i]B[i]|^2 / evalMaxApprox^2 " << std::setw(25)<< std::setiosflags(std::ios_base::right)<< vv
+			 <<" "<< vnum/(sqrt(vden)*sqrt(vv0))
+			 << " norm(B["<<j<<"])="<< vden <<std::endl;
+		
+		// change the criteria as evals are supposed to be sorted, all evals smaller(larger) than Nstop should have converged
+		if((vv<eresid*eresid) && (j == Nconv) ){
+		  Nconv+=SkipTest;
+		}
+	      }
+	      
+	      // test if we converged, if so, terminate
+	      t1=usecond()/1e6;
+	      std::cout<<GridLogMessage <<"IRL::convergence testing: "<<t1-t0<< "seconds"<<std::endl; t0=t1;
+	      
+	      std::cout<<GridLogMessage<<" #modes converged: "<<Nconv<<std::endl;
+	      
+	      if( Nconv>=Nstop || beta_k < betastp){
+		goto converged;
+	      }
+	      
+	      std::cout << GridLogMessage << "Rotate back" << std::endl;
+	      //B[j] +=Qt[k+_Nm*j] * _v[k]._odata[ss];
+	      {
+		Eigen::MatrixXd qm = Eigen::MatrixXd::Zero(Nk,Nk);
+		for (int k=0;k<Nk;k++)
+		  for (int j=0;j<Nk;j++)
+		    qm(j,k) = Qt[k+Nm*j];
+		GridStopWatch timeInv;
+		timeInv.Start();
+		Eigen::MatrixXd qmI = qm.inverse();
+		timeInv.Stop();
+		std::vector<RealD> QtI(Nm*Nm);
+		for (int k=0;k<Nk;k++)
+		  for (int j=0;j<Nk;j++)
+		    QtI[k+Nm*j] = qmI(j,k);
+		
+		RealD res_check_rotate_inverse = (qm*qmI - Eigen::MatrixXd::Identity(Nk,Nk)).norm(); // sqrt( |X|^2 )
+		assert(res_check_rotate_inverse < 1e-7);
+		evec.rotate(QtI,0,Nk,0,Nk,Nm);
+		
+		axpy(ev0_orig,-1.0,evec[0],ev0_orig);
+		std::cout << GridLogMessage << "Rotation done (in " << timeInv.Elapsed() << " = " << timeInv.useconds() << " us" <<
+		  ", error = " << res_check_rotate_inverse << 
+		  "); | evec[0] - evec[0]_orig | = " << ::sqrt(norm2(ev0_orig)) << std::endl;
+	      }
+	    }
+	  } else {
+	    std::cout << GridLogMessage << "iter < Nminres: do not yet test for convergence\n";
+	  } // end of iter loop
+	}
+
+	std::cout<<GridLogMessage<<"\n NOT converged.\n";
+	abort();
+	
+      converged:
+
+	if (SkipTest == 1) {
+	  eval = eval2;
+	} else {
+
+	  // test quickly
+	  for (int j=0;j<Nstop;j+=SkipTest) {
+	    std::cout<<GridLogMessage << "Eigenvalue[" << j << "] = " << eval2[j] << " (" << eval2_copy[j] << ")" << std::endl;
+	  }
+
+	  eval2_copy.resize(eval2.size());
+	  eval = eval2_copy;
+	}
+
+	evec.sortInPlace(eval,reverse);
+
+	{
+	  
+	 // test
+	 for (int j=0;j<Nstop;j++) {
+	   std::cout<<GridLogMessage << " |e[" << j << "]|^2 = " << norm2(evec[j]) << std::endl;
+	 }
+       }
+       
+       //_sort.push(eval,evec,Nconv);
+       //evec.sort(eval,Nconv);
+       
+       std::cout<<GridLogMessage << "\n Converged\n Summary :\n";
+       std::cout<<GridLogMessage << " -- Iterations  = "<< Nconv  << "\n";
+       std::cout<<GridLogMessage << " -- beta(k)     = "<< beta_k << "\n";
+       std::cout<<GridLogMessage << " -- Nconv       = "<< Nconv  << "\n";
+      }
+#endif
+
+    };
+
+}
+#endif
+
diff --git a/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockProjector.h b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockProjector.h
new file mode 100644
index 00000000..6becaa66
--- /dev/null
+++ b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockProjector.h
@@ -0,0 +1,143 @@
+namespace Grid { 
+
+/*
+  BlockProjector
+
+  If _HP_BLOCK_PROJECTORS_ is defined, we assume that _evec is a basis that is not
+  fully orthonormalized (to the precision of the coarse field) and we allow for higher-precision
+  coarse field than basis field.
+
+*/
+//#define _HP_BLOCK_PROJECTORS_
+
+template<typename Field>
+class BlockProjector {
+public:
+
+  BasisFieldVector<Field>& _evec;
+  BlockedGrid<Field>& _bgrid;
+
+  BlockProjector(BasisFieldVector<Field>& evec, BlockedGrid<Field>& bgrid) : _evec(evec), _bgrid(bgrid) {
+  }
+
+  void createOrthonormalBasis(RealD thres = 0.0) {
+
+    GridStopWatch sw;
+    sw.Start();
+
+    int cnt = 0;
+
+#pragma omp parallel shared(cnt)
+    {
+      int lcnt = 0;
+
+#pragma omp for
+      for (int b=0;b<_bgrid._o_blocks;b++) {
+	
+	for (int i=0;i<_evec._Nm;i++) {
+	  
+	  auto nrm0 = _bgrid.block_sp(b,_evec._v[i],_evec._v[i]);
+	  
+	  // |i> -= <j|i> |j>
+	  for (int j=0;j<i;j++) {
+	    _bgrid.block_caxpy(b,_evec._v[i],-_bgrid.block_sp(b,_evec._v[j],_evec._v[i]),_evec._v[j],_evec._v[i]);
+	  }
+	  
+	  auto nrm = _bgrid.block_sp(b,_evec._v[i],_evec._v[i]);
+	  
+	  auto eps = nrm/nrm0;
+	  if (Reduce(eps).real() < thres) {
+	    lcnt++;
+	  }
+	  
+	  // TODO: if norm is too small, remove this eigenvector/mark as not needed; in practice: set it to zero norm here and return a mask
+	  // that is then used later to decide not to write certain eigenvectors to disk (add a norm calculation before subtraction step and look at nrm/nrm0 < eps to decide)
+	  _bgrid.block_cscale(b,1.0 / sqrt(nrm),_evec._v[i]);
+	  
+	}
+	
+      }
+
+#pragma omp critical
+      {
+	cnt += lcnt;
+      }
+    }
+    sw.Stop();
+    std::cout << GridLogMessage << "Gram-Schmidt to create blocked basis took " << sw.Elapsed() << " (" << ((RealD)cnt / (RealD)_bgrid._o_blocks / (RealD)_evec._Nm) 
+	      << " below threshold)" << std::endl;
+
+  }
+
+  template<typename CoarseField>
+  void coarseToFine(const CoarseField& in, Field& out) {
+
+    out = zero;
+    out.checkerboard = _evec._v[0].checkerboard;
+
+    int Nbasis = sizeof(in._odata[0]._internal._internal) / sizeof(in._odata[0]._internal._internal[0]);
+    assert(Nbasis == _evec._Nm);
+    
+#pragma omp parallel for
+    for (int b=0;b<_bgrid._o_blocks;b++) {
+      for (int j=0;j<_evec._Nm;j++) {
+	_bgrid.block_caxpy(b,out,in._odata[b]._internal._internal[j],_evec._v[j],out);
+      }
+    }
+
+  }
+
+  template<typename CoarseField>
+  void fineToCoarse(const Field& in, CoarseField& out) {
+
+    out = zero;
+
+    int Nbasis = sizeof(out._odata[0]._internal._internal) / sizeof(out._odata[0]._internal._internal[0]);
+    assert(Nbasis == _evec._Nm);
+
+
+    Field tmp(_bgrid._grid);
+    tmp = in;
+    
+#pragma omp parallel for
+    for (int b=0;b<_bgrid._o_blocks;b++) {
+      for (int j=0;j<_evec._Nm;j++) {
+	// |rhs> -= <j|rhs> |j>
+	auto c = _bgrid.block_sp(b,_evec._v[j],tmp);
+	_bgrid.block_caxpy(b,tmp,-c,_evec._v[j],tmp); // may make this more numerically stable
+	out._odata[b]._internal._internal[j] = c;
+      }
+    }
+
+  }
+
+  template<typename CoarseField>
+    void deflateFine(BasisFieldVector<CoarseField>& _coef,const std::vector<RealD>& eval,int N,const Field& src_orig,Field& result) {
+    result = zero;
+    for (int i=0;i<N;i++) {
+      Field tmp(result._grid);
+      coarseToFine(_coef._v[i],tmp);
+      axpy(result,TensorRemove(innerProduct(tmp,src_orig)) / eval[i],tmp,result);
+    }
+  }
+
+  template<typename CoarseField>
+    void deflateCoarse(BasisFieldVector<CoarseField>& _coef,const std::vector<RealD>& eval,int N,const Field& src_orig,Field& result) {
+    CoarseField src_coarse(_coef._v[0]._grid);
+    CoarseField result_coarse = src_coarse;
+    result_coarse = zero;
+    fineToCoarse(src_orig,src_coarse);
+    for (int i=0;i<N;i++) {
+      axpy(result_coarse,TensorRemove(innerProduct(_coef._v[i],src_coarse)) / eval[i],_coef._v[i],result_coarse);
+    }
+    coarseToFine(result_coarse,result);
+  }
+
+  template<typename CoarseField>
+    void deflate(BasisFieldVector<CoarseField>& _coef,const std::vector<RealD>& eval,int N,const Field& src_orig,Field& result) {
+    // Deflation on coarse Grid is much faster, so use it by default.  Deflation on fine Grid is kept for legacy reasons for now.
+    deflateCoarse(_coef,eval,N,src_orig,result);
+  }
+
+};
+}
diff --git a/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockedGrid.h b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockedGrid.h
new file mode 100644
index 00000000..821272de
--- /dev/null
+++ b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockedGrid.h
@@ -0,0 +1,401 @@
+namespace Grid {
+
+template<typename Field>
+class BlockedGrid {
+public:
+  GridBase* _grid;
+  typedef typename Field::scalar_type  Coeff_t;
+  typedef typename Field::vector_type vCoeff_t;
+  
+  std::vector<int> _bs; // block size
+  std::vector<int> _nb; // number of blocks
+  std::vector<int> _l;  // local dimensions irrespective of cb
+  std::vector<int> _l_cb;  // local dimensions of checkerboarded vector
+  std::vector<int> _l_cb_o;  // local dimensions of inner checkerboarded vector
+  std::vector<int> _bs_cb; // block size in checkerboarded vector
+  std::vector<int> _nb_o; // number of blocks of simd o-sites
+
+  int _nd, _blocks, _cf_size, _cf_block_size, _cf_o_block_size, _o_blocks, _block_sites;
+  
+  BlockedGrid(GridBase* grid, const std::vector<int>& block_size) :
+    _grid(grid), _bs(block_size), _nd((int)_bs.size()), 
+      _nb(block_size), _l(block_size), _l_cb(block_size), _nb_o(block_size),
+      _l_cb_o(block_size), _bs_cb(block_size) {
+
+    _blocks = 1;
+    _o_blocks = 1;
+    _l = grid->FullDimensions();
+    _l_cb = grid->LocalDimensions();
+    _l_cb_o = grid->_rdimensions;
+
+    _cf_size = 1;
+    _block_sites = 1;
+    for (int i=0;i<_nd;i++) {
+      _l[i] /= grid->_processors[i];
+
+      assert(!(_l[i] % _bs[i])); // lattice must accommodate choice of blocksize
+
+      int r = _l[i] / _l_cb[i];
+      assert(!(_bs[i] % r)); // checkerboarding must accommodate choice of blocksize
+      _bs_cb[i] = _bs[i] / r;
+      _block_sites *= _bs_cb[i];
+      _nb[i] = _l[i] / _bs[i];
+      _nb_o[i] = _nb[i] / _grid->_simd_layout[i];
+      if (_nb[i] % _grid->_simd_layout[i]) { // simd must accommodate choice of blocksize
+	std::cout << GridLogMessage << "Problem: _nb[" << i << "] = " << _nb[i] << " _grid->_simd_layout[" << i << "] = " << _grid->_simd_layout[i] << std::endl;
+	assert(0);
+      }
+      _blocks *= _nb[i];
+      _o_blocks *= _nb_o[i];
+      _cf_size *= _l[i];
+    }
+
+    _cf_size *= 12 / 2;
+    _cf_block_size = _cf_size / _blocks;
+    _cf_o_block_size = _cf_size / _o_blocks;
+
+    std::cout << GridLogMessage << "BlockedGrid:" << std::endl;
+    std::cout << GridLogMessage << " _l     = " << _l << std::endl;
+    std::cout << GridLogMessage << " _l_cb     = " << _l_cb << std::endl;
+    std::cout << GridLogMessage << " _l_cb_o     = " << _l_cb_o << std::endl;
+    std::cout << GridLogMessage << " _bs    = " << _bs << std::endl;
+    std::cout << GridLogMessage << " _bs_cb    = " << _bs_cb << std::endl;
+
+    std::cout << GridLogMessage << " _nb    = " << _nb << std::endl;
+    std::cout << GridLogMessage << " _nb_o    = " << _nb_o << std::endl;
+    std::cout << GridLogMessage << " _blocks = " << _blocks << std::endl;
+    std::cout << GridLogMessage << " _o_blocks = " << _o_blocks << std::endl;
+    std::cout << GridLogMessage << " sizeof(vCoeff_t) = " << sizeof(vCoeff_t) << std::endl;
+    std::cout << GridLogMessage << " _cf_size = " << _cf_size << std::endl;
+    std::cout << GridLogMessage << " _cf_block_size = " << _cf_block_size << std::endl;
+    std::cout << GridLogMessage << " _block_sites = " << _block_sites << std::endl;
+    std::cout << GridLogMessage << " _grid->oSites() = " << _grid->oSites() << std::endl;
+
+    //    _grid->Barrier();
+    //abort();
+  }
+
+    void block_to_coor(int b, std::vector<int>& x0) {
+
+      std::vector<int> bcoor;
+      bcoor.resize(_nd);
+      x0.resize(_nd);
+      assert(b < _o_blocks);
+      Lexicographic::CoorFromIndex(bcoor,b,_nb_o);
+      int i;
+
+      for (i=0;i<_nd;i++) {
+	x0[i] = bcoor[i]*_bs_cb[i];
+      }
+
+      //std::cout << GridLogMessage << "Map block b -> " << x0 << std::endl;
+
+    }
+
+    void block_site_to_o_coor(const std::vector<int>& x0, std::vector<int>& coor, int i) {
+      Lexicographic::CoorFromIndex(coor,i,_bs_cb);
+      for (int j=0;j<_nd;j++)
+	coor[j] += x0[j];
+    }
+
+    int block_site_to_o_site(const std::vector<int>& x0, int i) {
+      std::vector<int> coor;  coor.resize(_nd);
+      block_site_to_o_coor(x0,coor,i);
+      Lexicographic::IndexFromCoor(coor,i,_l_cb_o);
+      return i;
+    }
+
+    vCoeff_t block_sp(int b, const Field& x, const Field& y) {
+
+      std::vector<int> x0;
+      block_to_coor(b,x0);
+
+      vCoeff_t ret = 0.0;
+      for (int i=0;i<_block_sites;i++) { // only odd sites
+	int ss = block_site_to_o_site(x0,i);
+	ret += TensorRemove(innerProduct(x._odata[ss],y._odata[ss]));
+      }
+
+      return ret;
+
+    }
+
+    vCoeff_t block_sp(int b, const Field& x, const std::vector< ComplexD >& y) {
+
+      std::vector<int> x0;
+      block_to_coor(b,x0);
+
+      constexpr int nsimd = sizeof(vCoeff_t) / sizeof(Coeff_t);
+      int lsize = _cf_o_block_size / _block_sites;
+
+      std::vector< ComplexD > ret(nsimd);
+      for (int i=0;i<nsimd;i++)
+	ret[i] = 0.0;
+
+      for (int i=0;i<_block_sites;i++) { // only odd sites
+	int ss = block_site_to_o_site(x0,i);
+
+	int n = lsize / nsimd;
+	for (int l=0;l<n;l++) {
+	  for (int j=0;j<nsimd;j++) {
+	    int t = lsize * i + l*nsimd + j;
+
+	    ret[j] += conjugate(((Coeff_t*)&x._odata[ss]._internal)[l*nsimd + j]) * y[t];
+	  }
+	}
+      }
+
+      vCoeff_t vret;
+      for (int i=0;i<nsimd;i++)
+	((Coeff_t*)&vret)[i] = (Coeff_t)ret[i];
+
+      return vret;
+
+    }
+
+    template<class T>
+      void vcaxpy(iScalar<T>& r,const vCoeff_t& a,const iScalar<T>& x,const iScalar<T>& y) {
+      vcaxpy(r._internal,a,x._internal,y._internal);
+    }
+
+    template<class T,int N>
+      void vcaxpy(iVector<T,N>& r,const vCoeff_t& a,const iVector<T,N>& x,const iVector<T,N>& y) {
+      for (int i=0;i<N;i++)
+	vcaxpy(r._internal[i],a,x._internal[i],y._internal[i]);
+    }
+
+    void vcaxpy(vCoeff_t& r,const vCoeff_t& a,const vCoeff_t& x,const vCoeff_t& y) {
+      r = a*x + y;
+    }
+
+    void block_caxpy(int b, Field& ret, const vCoeff_t& a, const Field& x, const Field& y) {
+
+      std::vector<int> x0;
+      block_to_coor(b,x0);
+
+      for (int i=0;i<_block_sites;i++) { // only odd sites
+	int ss = block_site_to_o_site(x0,i);
+	vcaxpy(ret._odata[ss],a,x._odata[ss],y._odata[ss]);
+      }
+
+    }
+
+    void block_caxpy(int b, std::vector< ComplexD >& ret, const vCoeff_t& a, const Field& x, const std::vector< ComplexD >& y) {
+      std::vector<int> x0;
+      block_to_coor(b,x0);
+
+      constexpr int nsimd = sizeof(vCoeff_t) / sizeof(Coeff_t);
+      int lsize = _cf_o_block_size / _block_sites;
+
+      for (int i=0;i<_block_sites;i++) { // only odd sites
+	int ss = block_site_to_o_site(x0,i);
+
+	int n = lsize / nsimd;
+	for (int l=0;l<n;l++) {
+	  vCoeff_t r = a* ((vCoeff_t*)&x._odata[ss]._internal)[l];
+
+	  for (int j=0;j<nsimd;j++) {
+	    int t = lsize * i + l*nsimd + j;
+	    ret[t] = y[t] + ((Coeff_t*)&r)[j];
+	  }
+	}
+      }
+
+    }
+
+    void block_set(int b, Field& ret, const std::vector< ComplexD >& x) {
+      std::vector<int> x0;
+      block_to_coor(b,x0);
+
+      int lsize = _cf_o_block_size / _block_sites;
+
+      for (int i=0;i<_block_sites;i++) { // only odd sites
+	int ss = block_site_to_o_site(x0,i);
+
+	for (int l=0;l<lsize;l++)
+	  ((Coeff_t*)&ret._odata[ss]._internal)[l] = (Coeff_t)x[lsize * i + l]; // convert precision
+      }
+
+    }
+
+    void block_get(int b, const Field& ret, std::vector< ComplexD >& x) {
+      std::vector<int> x0;
+      block_to_coor(b,x0);
+
+      int lsize = _cf_o_block_size / _block_sites;
+
+      for (int i=0;i<_block_sites;i++) { // only odd sites
+	int ss = block_site_to_o_site(x0,i);
+
+	for (int l=0;l<lsize;l++)
+	  x[lsize * i + l] = (ComplexD)((Coeff_t*)&ret._odata[ss]._internal)[l];
+      }
+
+    }
+
+    template<class T>
+    void vcscale(iScalar<T>& r,const vCoeff_t& a,const iScalar<T>& x) {
+      vcscale(r._internal,a,x._internal);
+    }
+
+    template<class T,int N>
+    void vcscale(iVector<T,N>& r,const vCoeff_t& a,const iVector<T,N>& x) {
+      for (int i=0;i<N;i++)
+	vcscale(r._internal[i],a,x._internal[i]);
+    }
+
+    void vcscale(vCoeff_t& r,const vCoeff_t& a,const vCoeff_t& x) {
+      r = a*x;
+    }
+
+    void block_cscale(int b, const vCoeff_t& a, Field& ret) {
+
+      std::vector<int> x0;
+      block_to_coor(b,x0);
+      
+      for (int i=0;i<_block_sites;i++) { // only odd sites
+	int ss = block_site_to_o_site(x0,i);
+	vcscale(ret._odata[ss],a,ret._odata[ss]);
+      }
+    }
+
+    void getCanonicalBlockOffset(int cb, std::vector<int>& x0) {
+      const int ndim = 5;
+      assert(_nb.size() == ndim);
+      std::vector<int> _nbc = { _nb[1], _nb[2], _nb[3], _nb[4], _nb[0] };
+      std::vector<int> _bsc = { _bs[1], _bs[2], _bs[3], _bs[4], _bs[0] };
+      x0.resize(ndim);
+
+      assert(cb >= 0);
+      assert(cb < _nbc[0]*_nbc[1]*_nbc[2]*_nbc[3]*_nbc[4]);
+
+      Lexicographic::CoorFromIndex(x0,cb,_nbc);
+      int i;
+
+      for (i=0;i<ndim;i++) {
+	x0[i] *= _bsc[i];
+      }
+
+      //if (cb < 2)
+      //	std::cout << GridLogMessage << "Map: " << cb << " To: " << x0 << std::endl;
+    }
+
+    void pokeBlockOfVectorCanonical(int cb,Field& v,const std::vector<float>& buf) {
+      std::vector<int> _bsc = { _bs[1], _bs[2], _bs[3], _bs[4], _bs[0] };
+      std::vector<int> ldim = v._grid->LocalDimensions();
+      std::vector<int> cldim = { ldim[1], ldim[2], ldim[3], ldim[4], ldim[0] };
+      const int _nbsc = _bs_cb[0]*_bs_cb[1]*_bs_cb[2]*_bs_cb[3]*_bs_cb[4];
+      // take canonical block cb of v and put it in canonical ordering in buf
+      std::vector<int> cx0;
+      getCanonicalBlockOffset(cb,cx0);
+
+#pragma omp parallel
+      {
+	std::vector<int> co0,cl0;
+	co0=cx0; cl0=cx0;
+
+#pragma omp for
+	for (int i=0;i<_nbsc;i++) {
+	  Lexicographic::CoorFromIndex(co0,2*i,_bsc); // 2* for eo
+	  for (int j=0;j<(int)_bsc.size();j++)
+	    cl0[j] = cx0[j] + co0[j];
+	  
+	  std::vector<int> l0 = { cl0[4], cl0[0], cl0[1], cl0[2], cl0[3] };
+	  int oi = v._grid->oIndex(l0);
+	  int ii = v._grid->iIndex(l0);
+	  int lti = i;
+
+	  //if (cb < 2 && i<2)
+	  //  std::cout << GridLogMessage << "Map: " << cb << ", " << i << " To: " << cl0 << ", " << cx0 << ", " << oi << ", " << ii << std::endl;
+	  
+	  for (int s=0;s<4;s++)
+	    for (int c=0;c<3;c++) {
+	      Coeff_t& ld = ((Coeff_t*)&v._odata[oi]._internal._internal[s]._internal[c])[ii];
+	      int ti = 12*lti + 3*s + c;
+	      ld = Coeff_t(buf[2*ti+0], buf[2*ti+1]);
+	    }
+	}
+      }
+    }
+
+    void peekBlockOfVectorCanonical(int cb,const Field& v,std::vector<float>& buf) {
+      std::vector<int> _bsc = { _bs[1], _bs[2], _bs[3], _bs[4], _bs[0] };
+      std::vector<int> ldim = v._grid->LocalDimensions();
+      std::vector<int> cldim = { ldim[1], ldim[2], ldim[3], ldim[4], ldim[0] };
+      const int _nbsc = _bs_cb[0]*_bs_cb[1]*_bs_cb[2]*_bs_cb[3]*_bs_cb[4];
+      // take canonical block cb of v and put it in canonical ordering in buf
+      std::vector<int> cx0;
+      getCanonicalBlockOffset(cb,cx0);
+
+      buf.resize(_cf_block_size * 2);
+
+#pragma omp parallel
+      {
+	std::vector<int> co0,cl0;
+	co0=cx0; cl0=cx0;
+
+#pragma omp for
+	for (int i=0;i<_nbsc;i++) {
+	  Lexicographic::CoorFromIndex(co0,2*i,_bsc); // 2* for eo
+	  for (int j=0;j<(int)_bsc.size();j++)
+	    cl0[j] = cx0[j] + co0[j];
+	  
+	  std::vector<int> l0 = { cl0[4], cl0[0], cl0[1], cl0[2], cl0[3] };
+	  int oi = v._grid->oIndex(l0);
+	  int ii = v._grid->iIndex(l0);
+	  int lti = i;
+	  
+	  //if (cb < 2 && i<2)
+	  //  std::cout << GridLogMessage << "Map: " << cb << ", " << i << " To: " << cl0 << ", " << cx0 << ", " << oi << ", " << ii << std::endl;
+
+	  for (int s=0;s<4;s++)
+	    for (int c=0;c<3;c++) {
+	      Coeff_t& ld = ((Coeff_t*)&v._odata[oi]._internal._internal[s]._internal[c])[ii];
+	      int ti = 12*lti + 3*s + c;
+	      buf[2*ti+0] = ld.real();
+	      buf[2*ti+1] = ld.imag();
+	    }
+	}
+      }
+    }
+
+    int globalToLocalCanonicalBlock(int slot,const std::vector<int>& src_nodes,int nb) {
+      // processor coordinate
+      int _nd = (int)src_nodes.size();
+      std::vector<int> _src_nodes = src_nodes;
+      std::vector<int> pco(_nd);
+      Lexicographic::CoorFromIndex(pco,slot,_src_nodes);
+      std::vector<int> cpco = { pco[1], pco[2], pco[3], pco[4], pco[0] };
+
+      // get local block
+      std::vector<int> _nbc = { _nb[1], _nb[2], _nb[3], _nb[4], _nb[0] };
+      assert(_nd == 5);
+      std::vector<int> c_src_local_blocks(_nd);
+      for (int i=0;i<_nd;i++) {
+	assert(_grid->_fdimensions[i] % (src_nodes[i] * _bs[i]) == 0);
+	c_src_local_blocks[(i+4) % 5] = _grid->_fdimensions[i] / src_nodes[i] / _bs[i];
+      }
+      std::vector<int> cbcoor(_nd); // coordinate of block in slot in canonical form
+      Lexicographic::CoorFromIndex(cbcoor,nb,c_src_local_blocks);
+
+      // cpco, cbcoor
+      std::vector<int> clbcoor(_nd);
+      for (int i=0;i<_nd;i++) {
+	int cgcoor = cpco[i] * c_src_local_blocks[i] + cbcoor[i]; // global block coordinate
+	int pcoor = cgcoor / _nbc[i]; // processor coordinate in my Grid
+	int tpcoor = _grid->_processor_coor[(i+1)%5];
+	if (pcoor != tpcoor)
+	  return -1;
+	clbcoor[i] = cgcoor - tpcoor * _nbc[i]; // canonical local block coordinate for canonical dimension i
+      }
+
+      int lnb;
+      Lexicographic::IndexFromCoor(clbcoor,lnb,_nbc);
+      //std::cout << "Mapped slot = " << slot << " nb = " << nb << " to " << lnb << std::endl;
+      return lnb;
+    }
+
+
+ };
+
+}
diff --git a/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldBasisVector.h b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldBasisVector.h
new file mode 100644
index 00000000..e715fc25
--- /dev/null
+++ b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldBasisVector.h
@@ -0,0 +1,163 @@
+namespace Grid { 
+
+template<class Field>
+class BasisFieldVector {
+ public:
+  int _Nm;
+
+  typedef typename Field::scalar_type Coeff_t;
+  typedef typename Field::vector_type vCoeff_t;
+  typedef typename Field::vector_object vobj;
+  typedef typename vobj::scalar_object sobj;
+
+  std::vector<Field> _v; // _Nfull vectors
+
+  void report(int n,GridBase* value) {
+
+    std::cout << GridLogMessage << "BasisFieldVector allocated:\n";
+    std::cout << GridLogMessage << " Delta N = " << n << "\n";
+    std::cout << GridLogMessage << " Size of full vectors (size) = " << 
+      ((double)n*sizeof(vobj)*value->oSites() / 1024./1024./1024.) << " GB\n";
+    std::cout << GridLogMessage << " Size = " << _v.size() << " Capacity = " << _v.capacity() << std::endl;
+
+    value->Barrier();
+
+    if (value->IsBoss()) {
+      system("cat /proc/meminfo");
+    }
+
+    value->Barrier();
+
+  }
+
+  BasisFieldVector(int Nm,GridBase* value) : _Nm(Nm), _v(Nm,value) {
+    report(Nm,value);
+  }
+  
+  ~BasisFieldVector() {
+  }
+
+  Field& operator[](int i) {
+    return _v[i];
+  }
+
+  void orthogonalize(Field& w, int k) {
+    for(int j=0; j<k; ++j){
+      Coeff_t ip = (Coeff_t)innerProduct(_v[j],w);
+      w = w - ip*_v[j];
+    }
+  }
+
+  void rotate(std::vector<RealD>& Qt,int j0, int j1, int k0,int k1,int Nm) {
+    
+    GridBase* grid = _v[0]._grid;
+      
+#pragma omp parallel
+    {
+      std::vector < vobj > B(Nm);
+      
+#pragma omp for
+      for(int ss=0;ss < grid->oSites();ss++){
+	for(int j=j0; j<j1; ++j) B[j]=0.;
+	
+	for(int j=j0; j<j1; ++j){
+	  for(int k=k0; k<k1; ++k){
+	    B[j] +=Qt[k+Nm*j] * _v[k]._odata[ss];
+	  }
+	}
+	for(int j=j0; j<j1; ++j){
+	  _v[j]._odata[ss] = B[j];
+	}
+      }
+    }
+
+  }
+
+  size_t size() const {
+    return _Nm;
+  }
+
+  void resize(int n) {
+    if (n > _Nm)
+      _v.reserve(n);
+    
+    _v.resize(n,_v[0]._grid);
+
+    if (n < _Nm)
+      _v.shrink_to_fit();
+
+    report(n - _Nm,_v[0]._grid);
+
+    _Nm = n;
+  }
+
+  std::vector<int> getIndex(std::vector<RealD>& sort_vals) {
+
+    std::vector<int> idx(sort_vals.size());
+    iota(idx.begin(), idx.end(), 0);
+
+    // sort indexes based on comparing values in v
+    sort(idx.begin(), idx.end(),
+	 [&sort_vals](int i1, int i2) {return ::fabs(sort_vals[i1]) < ::fabs(sort_vals[i2]);});
+
+    return idx;
+  }
+
+  void reorderInPlace(std::vector<RealD>& sort_vals, std::vector<int>& idx) {
+    GridStopWatch gsw;
+    gsw.Start();
+
+    int nswaps = 0;
+    for (size_t i=0;i<idx.size();i++) {
+      if (idx[i] != i) {
+
+	// find proper place (this could be done in logarithmic time, don't bother for now)
+	size_t j;
+	for (j=i;j<idx.size();j++)
+	  if (idx[j]==i)
+	    break;
+	assert(j!=idx.size());
+	
+	Field _t(_v[0]._grid);
+	_t = _v[idx[j]];
+	_v[idx[j]] = _v[idx[i]];
+	_v[idx[i]] = _t;
+
+	RealD _td = sort_vals[idx[j]];
+	sort_vals[idx[j]] = sort_vals[idx[i]];
+	sort_vals[idx[i]] = _td;
+
+	int _tt = idx[i];
+	idx[i] = idx[j];
+	idx[j] = _tt;
+	
+	nswaps++;
+      }
+    }
+
+    // sort values
+    gsw.Stop();
+    std::cout << GridLogMessage << "Sorted eigenspace in place in " << gsw.Elapsed() << " using " << nswaps << " swaps" << std::endl;
+  }
+
+  void sortInPlace(std::vector<RealD>& sort_vals, bool reverse) {
+
+    std::vector<int> idx = getIndex(sort_vals);
+    if (reverse)
+      std::reverse(idx.begin(), idx.end());
+
+    reorderInPlace(sort_vals,idx);
+
+  }
+
+  void deflate(const std::vector<RealD>& eval,const Field& src_orig,Field& result) {
+    result = zero;
+    int N = (int)_v.size();
+    for (int i=0;i<N;i++) {
+      Field& tmp = _v[i];
+      axpy(result,TensorRemove(innerProduct(tmp,src_orig)) / eval[i],tmp,result);
+    }
+  }
+
+ }; 
+}
diff --git a/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldVectorIO.h b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldVectorIO.h
new file mode 100644
index 00000000..344e48fe
--- /dev/null
+++ b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldVectorIO.h
@@ -0,0 +1,1085 @@
+namespace Grid { 
+
+ namespace FieldVectorIO {
+
+   // zlib's crc32 gets 0.4 GB/s on KNL single thread
+   // below gets 4.8 GB/s
+   static uint32_t crc32_threaded(unsigned char* data, int64_t len, uint32_t previousCrc32 = 0) {
+
+     // crc32 of zlib was incorrect for very large sizes, so do it block-wise
+     uint32_t crc = previousCrc32;
+     off_t blk = 0;
+     off_t step = 1024*1024*1024;
+     while (len > step) {
+       crc = crc32(crc,&data[blk],step);
+       blk += step;
+       len -= step;
+     }
+
+     crc = crc32(crc,&data[blk],len);
+     return crc;
+
+   }
+ 
+   static int get_bfm_index( int* pos, int co, int* s ) {
+     
+     int ls = s[0];
+     int NtHalf = s[4] / 2;
+     int simd_coor = pos[4] / NtHalf;
+     int regu_coor = (pos[1] + s[1] * (pos[2] + s[2] * ( pos[3] + s[3] * (pos[4] % NtHalf) ) )) / 2;
+     
+     return regu_coor * ls * 48 + pos[0] * 48 + co * 4 + simd_coor * 2;
+   }
+   
+   static void get_read_geometry(const GridBase* _grid,const std::vector<int>& cnodes,
+			  std::map<int, std::vector<int> >& slots, 
+			  std::vector<int>& slot_lvol,
+			  std::vector<int>& lvol,
+			  int64_t& slot_lsites,int& ntotal) {
+
+     int _nd = (int)cnodes.size();
+     std::vector<int> nodes = cnodes;
+
+     slots.clear();
+     slot_lvol.clear();
+     lvol.clear();
+
+     int i;
+     ntotal = 1;
+     int64_t lsites = 1;
+     slot_lsites = 1;
+     for (i=0;i<_nd;i++) {
+       assert(_grid->_fdimensions[i] % nodes[i] == 0);
+       slot_lvol.push_back(_grid->_fdimensions[i] / nodes[i]);
+       lvol.push_back(_grid->_fdimensions[i] / _grid->_processors[i]);
+       lsites *= lvol.back();
+       slot_lsites *= slot_lvol.back();
+       ntotal *= nodes[i];
+     }
+
+     std::vector<int> lcoor, gcoor, scoor;
+     lcoor.resize(_nd); gcoor.resize(_nd);  scoor.resize(_nd);
+     
+     // create mapping of indices to slots
+     for (int lidx = 0; lidx < lsites; lidx++) {
+       Lexicographic::CoorFromIndex(lcoor,lidx,lvol);
+       for (int i=0;i<_nd;i++) {
+	 gcoor[i] = lcoor[i] + _grid->_processor_coor[i]*lvol[i];
+	 scoor[i] = gcoor[i] / slot_lvol[i];
+       }
+       int slot;
+       Lexicographic::IndexFromCoor(scoor,slot,nodes);
+       auto sl = slots.find(slot);
+       if (sl == slots.end())
+	 slots[slot] = std::vector<int>();
+       slots[slot].push_back(lidx);
+     }
+   }
+   
+   static void canonical_block_to_coarse_coordinates(GridBase* _coarsegrid,int nb,int& ii,int& oi) {
+      // canonical nb needs to be mapped in a coordinate on my coarsegrid (ii,io)
+      std::vector<int> _l = _coarsegrid->LocalDimensions();
+      std::vector<int> _cl = { _l[1], _l[2], _l[3], _l[4], _l[0] };
+      std::vector<int> _cc(_l.size());
+      Lexicographic::CoorFromIndex(_cc,nb,_cl);
+      std::vector<int> _c = { _cc[4], _cc[0], _cc[1], _cc[2], _cc[3] };
+      ii = _coarsegrid->iIndex(_c);
+      oi = _coarsegrid->oIndex(_c);
+    }
+
+    template<typename Field>
+     static bool read_argonne(BasisFieldVector<Field>& ret,const char* dir, const std::vector<int>& cnodes) {
+
+     GridBase* _grid = ret._v[0]._grid;
+
+     std::map<int, std::vector<int> > slots;
+     std::vector<int> slot_lvol, lvol;
+     int64_t slot_lsites;
+     int ntotal;
+     get_read_geometry(_grid,cnodes,
+		       slots,slot_lvol,lvol,slot_lsites,
+		       ntotal);
+     int _nd = (int)lvol.size();
+
+     // this is slow code to read the argonne file format for debugging purposes
+     int nperdir = ntotal / 32;
+     if (nperdir < 1)
+       nperdir=1;
+     std::cout << GridLogMessage << " Read " << dir << " nodes = " << cnodes << std::endl;
+     std::cout << GridLogMessage << " lvol = " << lvol << std::endl;
+     
+     // for error messages
+     char hostname[1024];
+     gethostname(hostname, 1024);
+
+     // now load one slot at a time and fill the vector
+     for (auto sl=slots.begin();sl!=slots.end();sl++) {
+       std::vector<int>& idx = sl->second;
+       int slot = sl->first;
+       std::vector<float> rdata;
+       
+       char buf[4096];
+       
+       sprintf(buf,"%s/checksums.txt",dir); printf("read_argonne: Reading from %s\n",buf);
+       FILE* f = fopen(buf,"rt");
+       if (!f) {
+	 fprintf(stderr,"Node %s cannot read %s\n",hostname,buf); fflush(stderr);
+	 return false;
+       }
+       
+       for (int l=0;l<3+slot;l++)
+	 fgets(buf,sizeof(buf),f);
+       uint32_t crc_exp = strtol(buf, NULL, 16);
+       fclose(f);
+       
+       // load one slot vector
+       sprintf(buf,"%s/%2.2d/%10.10d",dir,slot/nperdir,slot);
+       f = fopen(buf,"rb");
+       if (!f) {
+	 fprintf(stderr,"Node %s cannot read %s\n",hostname,buf); fflush(stderr);
+	 return false;
+       }
+       
+       fseeko(f,0,SEEK_END);
+       off_t total_size = ftello(f);
+       fseeko(f,0,SEEK_SET);
+       
+       int64_t size = slot_lsites / 2 * 24*4;
+       rdata.resize(size);
+       
+       assert(total_size % size == 0);
+       
+       int _Nfull = total_size / size;
+       ret._v.resize(_Nfull,ret._v[0]);
+       ret._Nm = _Nfull;
+       
+       uint32_t crc = 0x0;
+       GridStopWatch gsw,gsw2;
+       for (int nev = 0;nev < _Nfull;nev++) {
+	 
+	 gsw.Start();
+	 assert(fread(&rdata[0],size,1,f) == 1);
+	 gsw.Stop();
+	 
+	 gsw2.Start();
+	 crc = crc32_threaded((unsigned char*)&rdata[0],size,crc);
+	 gsw2.Stop();
+	 
+	 for (int i=0;i<size/4;i++) {
+	   char* c = (char*)&rdata[i];
+	   char tmp; int j;
+	   for (j=0;j<2;j++) {
+	     tmp = c[j]; c[j] = c[3-j]; c[3-j] = tmp;
+	   }
+	 }
+	 
+	 // loop
+	 ret._v[nev].checkerboard = Odd;
+#pragma omp parallel 
+	 {
+	   
+	   std::vector<int> lcoor, gcoor, scoor, slcoor;
+	   lcoor.resize(_nd); gcoor.resize(_nd); 
+	   slcoor.resize(_nd); scoor.resize(_nd);
+	   
+#pragma omp for
+	   for (int64_t lidx = 0; lidx < idx.size(); lidx++) {
+	     int llidx = idx[lidx];
+	     Lexicographic::CoorFromIndex(lcoor,llidx,lvol);
+	     for (int i=0;i<_nd;i++) {
+	       gcoor[i] = lcoor[i] + _grid->_processor_coor[i]*lvol[i];
+	       scoor[i] = gcoor[i] / slot_lvol[i];
+	       slcoor[i] = gcoor[i] - scoor[i]*slot_lvol[i];
+	     }
+	     
+	     if ((lcoor[1]+lcoor[2]+lcoor[3]+lcoor[4]) % 2 == 1) {
+	       // poke 
+	       iScalar<iVector<iVector<ComplexF, 3>, 4> > sc;
+	       for (int s=0;s<4;s++)
+		 for (int c=0;c<3;c++)
+		   sc()(s)(c) = *(std::complex<float>*)&rdata[get_bfm_index(&slcoor[0],c+s*3, &slot_lvol[0] )];
+	       
+	       pokeLocalSite(sc,ret._v[nev],lcoor);
+	     }
+	     
+	   }
+	 }
+       }
+       
+       fclose(f);      
+       std::cout << GridLogMessage << "Loading slot " << slot << " with " << idx.size() << " points and " 
+		 << _Nfull << " vectors in "
+		 << gsw.Elapsed() << " at " 
+		 << ( (double)size * _Nfull / 1024./1024./1024. / gsw.useconds()*1000.*1000. )
+		 << " GB/s " << " crc32 = " << std::hex << crc << " crc32_expected = " << crc_exp << std::dec
+		 << " computed at "
+		 << ( (double)size * _Nfull / 1024./1024./1024. / gsw2.useconds()*1000.*1000. )
+		 << " GB/s "
+		 << std::endl;
+       
+       assert(crc == crc_exp);
+     }
+
+     _grid->Barrier();
+     std::cout << GridLogMessage  << "Loading complete" << std::endl;
+     
+     return true;
+   }
+
+   template<typename Field>
+     static bool read_argonne(BasisFieldVector<Field>& ret,const char* dir) {
+
+
+     GridBase* _grid = ret._v[0]._grid;
+
+     char buf[4096];
+     sprintf(buf,"%s/nodes.txt",dir);
+     FILE* f = fopen(buf,"rt");
+     if (!f) {
+       if (_grid->IsBoss()) {
+	 fprintf(stderr,"Attempting to load eigenvectors without secifying node layout failed due to absence of nodes.txt\n");
+	 fflush(stderr);
+       }
+       return false;
+     }
+
+
+     std::vector<int> nodes((int)_grid->_processors.size());
+     for (int i =0;i<(int)_grid->_processors.size();i++)
+       assert(fscanf(f,"%d\n",&nodes[i])==1);
+     fclose(f);
+
+     return read_argonne(ret,dir,nodes);
+   }
+
+   static void flush_bytes(FILE* f, std::vector<char>& fbuf) {
+     if (fbuf.size()) {
+
+       if (fwrite(&fbuf[0],fbuf.size(),1,f) != 1) {
+	 fprintf(stderr,"Write failed of %g GB!\n",(double)fbuf.size() / 1024./1024./1024.);
+	 exit(2);
+       }
+
+       fbuf.resize(0);
+
+     }
+   }
+
+   static void write_bytes(void* buf, int64_t s, FILE* f, std::vector<char>& fbuf, uint32_t& crc) {
+     static double data_counter = 0.0;
+     static GridStopWatch gsw_crc, gsw_flush1,gsw_flush2,gsw_write,gsw_memcpy;
+     if (s == 0)
+       return;
+
+     // checksum
+     gsw_crc.Start();
+     crc = crc32_threaded((unsigned char*)buf,s,crc);
+     gsw_crc.Stop();
+
+     if (s > fbuf.capacity()) {
+       // cannot buffer this, so first flush current buffer contents and then write this directly to file
+       gsw_flush1.Start();
+       flush_bytes(f,fbuf);
+       gsw_flush1.Stop();
+
+       gsw_write.Start();
+       if (fwrite(buf,s,1,f) != 1) {
+	 fprintf(stderr,"Write failed of %g GB!\n",(double)s / 1024./1024./1024.);
+	 exit(2);
+       }
+       gsw_write.Stop();
+
+     }
+
+     // no room left in buffer, flush to disk
+     if (fbuf.size() + s > fbuf.capacity()) {
+       gsw_flush2.Start();
+       flush_bytes(f,fbuf);
+       gsw_flush2.Stop();
+     }
+
+     // then fill buffer again
+     {
+       gsw_memcpy.Start();
+       size_t t = fbuf.size();
+       fbuf.resize(t + s);
+       memcpy(&fbuf[t],buf,s);
+       gsw_memcpy.Stop();
+     }
+
+     data_counter += (double)s;
+     if (data_counter > 1024.*1024.*20.) {
+       std::cout << GridLogMessage << "Writing " << ((double)data_counter / 1024./1024./1024.) << " GB at"
+	 " crc = " << gsw_crc.Elapsed() << " flush1 = " << gsw_flush1.Elapsed() << " flush2 = " << gsw_flush2.Elapsed() <<
+	 " write = " << gsw_write.Elapsed() << " memcpy = " << gsw_memcpy.Elapsed() << std::endl;
+       data_counter = 0.0;
+       gsw_crc.Reset();
+       gsw_write.Reset();
+       gsw_memcpy.Reset();
+       gsw_flush1.Reset();
+       gsw_flush2.Reset();
+     }
+   }
+
+   static void write_floats(FILE* f, std::vector<char>& fbuf, uint32_t& crc, float* buf, int64_t n) {
+     write_bytes(buf,n*sizeof(float),f,fbuf,crc);
+   }
+
+   static void read_floats(char* & ptr, float* out, int64_t n) {
+     float* in = (float*)ptr;
+     ptr += 4*n;
+
+     for (int64_t i=0;i<n;i++)
+       out[i] = in[i];
+   }
+
+   static int fp_map(float in, float min, float max, int N) {
+     // Idea:
+     //
+     // min=-6
+     // max=6
+     //
+     // N=1
+     // [-6,0] -> 0, [0,6] -> 1;  reconstruct 0 -> -3, 1-> 3
+     //
+     // N=2
+     // [-6,-2] -> 0, [-2,2] -> 1, [2,6] -> 2;  reconstruct 0 -> -4, 1->0, 2->4
+     int ret =  (int) ( (float)(N+1) * ( (in - min) / (max - min) ) );
+     if (ret == N+1) {
+       ret = N;
+     }
+     return ret;
+   }
+
+   static float fp_unmap(int val, float min, float max, int N) {
+     return min + (float)(val + 0.5) * (max - min)  / (float)( N + 1 );
+   }
+
+#define SHRT_UMAX 65535
+#define FP16_BASE 1.4142135623730950488
+#define FP16_COEF_EXP_SHARE_FLOATS 10
+   static float unmap_fp16_exp(unsigned short e) {
+     float de = (float)((int)e - SHRT_UMAX / 2);
+     return ::pow( FP16_BASE, de );
+   }
+
+   // can assume that v >=0 and need to guarantee that unmap_fp16_exp(map_fp16_exp(v)) >= v
+   static unsigned short map_fp16_exp(float v) {
+     // float has exponents 10^{-44.85} .. 10^{38.53}
+     int exp = (int)ceil(::log(v) / ::log(FP16_BASE)) + SHRT_UMAX / 2;
+     if (exp < 0 || exp > SHRT_UMAX) {
+       fprintf(stderr,"Error in map_fp16_exp(%g,%d)\n",v,exp);
+       exit(3);
+     }
+
+     return (unsigned short)exp;
+   }
+  
+   template<typename OPT>
+     static void read_floats_fp16(char* & ptr, OPT* out, int64_t n, int nsc) {
+
+     int64_t nsites = n / nsc;
+     if (n % nsc) {
+       fprintf(stderr,"Invalid size in write_floats_fp16\n");
+       exit(4);
+     }
+
+     unsigned short* in = (unsigned short*)ptr;
+     ptr += 2*(n+nsites);
+
+     // do for each site
+     for (int64_t site = 0;site<nsites;site++) {
+
+       OPT* ev = &out[site*nsc];
+
+       unsigned short* bptr = &in[site*(nsc + 1)];
+
+       unsigned short exp = *bptr++;
+       OPT max = unmap_fp16_exp(exp);
+       OPT min = -max;
+
+       for (int i=0;i<nsc;i++) {
+	 ev[i] = fp_unmap( *bptr++, min, max, SHRT_UMAX );
+       }
+
+     }
+
+   }
+
+   template<typename OPT>
+     static void write_floats_fp16(FILE* f, std::vector<char>& fbuf, uint32_t& crc, OPT* in, int64_t n, int nsc) {
+
+     int64_t nsites = n / nsc;
+     if (n % nsc) {
+       fprintf(stderr,"Invalid size in write_floats_fp16\n");
+       exit(4);
+     }
+
+     unsigned short* buf = (unsigned short*)malloc( sizeof(short) * (n + nsites) );
+     if (!buf) {
+       fprintf(stderr,"Out of mem\n");
+       exit(1);
+     }
+
+     // do for each site
+#pragma omp parallel for
+     for (int64_t site = 0;site<nsites;site++) {
+
+       OPT* ev = &in[site*nsc];
+
+       unsigned short* bptr = &buf[site*(nsc + 1)];
+
+       OPT max = fabs(ev[0]);
+       OPT min;
+
+       for (int i=0;i<nsc;i++) {
+	 if (fabs(ev[i]) > max)
+	   max = fabs(ev[i]);
+       }
+
+       unsigned short exp = map_fp16_exp(max);
+       max = unmap_fp16_exp(exp);
+       min = -max;
+
+       *bptr++ = exp;
+
+       for (int i=0;i<nsc;i++) {
+	 int val = fp_map( ev[i], min, max, SHRT_UMAX );
+	 if (val < 0 || val > SHRT_UMAX) {
+	   fprintf(stderr,"Assert failed: val = %d (%d), ev[i] = %.15g, max = %.15g, exp = %d\n",val,SHRT_UMAX,ev[i],max,(int)exp);
+	   exit(48);
+	 }
+	 *bptr++ = (unsigned short)val;
+       }
+
+     }
+
+     write_bytes(buf,sizeof(short)*(n + nsites),f,fbuf,crc);
+
+     free(buf);
+   }
+
+   template<typename Field,typename CoarseField>
+     static bool read_compressed_vectors(const char* dir,BlockProjector<Field>& pr,BasisFieldVector<CoarseField>& coef, int ngroups = 1) {
+
+     const BasisFieldVector<Field>& basis = pr._evec;
+     GridBase* _grid = basis._v[0]._grid;
+
+     // for error messages
+     char hostname[1024];
+     gethostname(hostname, 1024);
+
+     std::cout << GridLogMessage << "Ready on host " << hostname << " with " << ngroups << " reader groups" << std::endl;
+
+     // first read metadata
+     char buf[4096];
+     sprintf(buf,"%s/metadata.txt",dir);
+
+     std::vector<uint32_t> s,b,nb,nn,crc32;
+     s.resize(5); b.resize(5); nb.resize(5); nn.resize(5);
+     uint32_t neig, nkeep, nkeep_single, blocks, _FP16_COEF_EXP_SHARE_FLOATS;
+     uint32_t nprocessors = 1;
+
+     FILE* f = 0;
+     uint32_t status = 0;
+     if (_grid->IsBoss()) {
+       f = fopen(buf,"rb");
+       status=f ? 1 : 0;
+     }
+     _grid->GlobalSum(status);
+     std::cout << GridLogMessage << "Read params status " << status << std::endl;
+
+     if (!status) {
+       return false;
+     }
+
+#define _IRL_READ_INT(buf,p) if (f) { assert(fscanf(f,buf,p)==1); } else { *(p) = 0; } _grid->GlobalSum(*(p));
+
+     for (int i=0;i<5;i++) {
+       sprintf(buf,"s[%d] = %%d\n",i);
+       _IRL_READ_INT(buf,&s[(i+1)%5]);
+     }
+     for (int i=0;i<5;i++) {
+       sprintf(buf,"b[%d] = %%d\n",i);
+       _IRL_READ_INT(buf,&b[(i+1)%5]);
+     }
+     for (int i=0;i<5;i++) {
+       sprintf(buf,"nb[%d] = %%d\n",i);
+       _IRL_READ_INT(buf,&nb[(i+1)%5]);
+     }
+     _IRL_READ_INT("neig = %d\n",&neig);
+     _IRL_READ_INT("nkeep = %d\n",&nkeep);
+     _IRL_READ_INT("nkeep_single = %d\n",&nkeep_single);
+     _IRL_READ_INT("blocks = %d\n",&blocks);
+     _IRL_READ_INT("FP16_COEF_EXP_SHARE_FLOATS = %d\n",&_FP16_COEF_EXP_SHARE_FLOATS);
+
+     for (int i=0;i<5;i++) {
+       assert(_grid->FullDimensions()[i] % s[i] == 0);
+       nn[i] = _grid->FullDimensions()[i] / s[i];
+       nprocessors *= nn[i];
+     }
+
+     std::cout << GridLogMessage << "Reading data that was generated on node-layout " << nn << std::endl;
+
+     crc32.resize(nprocessors);
+     for (int i =0;i<nprocessors;i++) {
+       sprintf(buf,"crc32[%d] = %%X\n",i);
+       _IRL_READ_INT(buf,&crc32[i]);
+     }
+
+#undef _IRL_READ_INT
+
+     if (f)
+       fclose(f);
+
+     // allocate memory
+     assert(std::equal(pr._bgrid._bs.begin(),pr._bgrid._bs.end(),b.begin())); // needs to be the same for now
+     assert(pr._evec.size() == nkeep); // needs to be the same since this is compile-time fixed
+     coef.resize(neig);
+
+
+     // now get read geometry
+     std::map<int, std::vector<int> > slots;
+     std::vector<int> slot_lvol, lvol;
+     int64_t slot_lsites;
+     int ntotal;
+     std::vector<int> _nn(nn.begin(),nn.end());
+     get_read_geometry(_grid,_nn,
+		       slots,slot_lvol,lvol,slot_lsites,
+		       ntotal);
+     int _nd = (int)lvol.size();
+
+     // types
+     typedef typename Field::scalar_type Coeff_t;
+     typedef typename CoarseField::scalar_type CoeffCoarse_t;
+
+     // slot layout
+     int nperdir = ntotal / 32;
+     if (nperdir < 1)
+       nperdir=1;
+
+     // add read groups
+     for (int ngroup=0;ngroup<ngroups;ngroup++) {
+
+       bool action = _grid->ThisRank() % ngroups == ngroup;
+       
+       std::cout << GridLogMessage << "Reading in group " << ngroup << " / " << ngroups << std::endl;
+
+       // load all necessary slots and store them appropriately
+       for (auto sl=slots.begin();sl!=slots.end();sl++) {
+
+	 std::vector<int>& idx = sl->second;
+	 int slot = sl->first;
+	 std::vector<float> rdata;
+       
+	 char buf[4096];
+       
+	 if (action) {
+	   // load one slot vector
+	   sprintf(buf,"%s/%2.2d/%10.10d.compressed",dir,slot/nperdir,slot);
+	   f = fopen(buf,"rb");
+	   if (!f) {
+	     fprintf(stderr,"Node %s cannot read %s\n",hostname,buf); fflush(stderr);
+	     return false;
+	   }
+	 }
+
+	 uint32_t crc = 0x0;
+	 off_t size;
+	   
+	 GridStopWatch gsw;
+	 _grid->Barrier();
+	 gsw.Start();
+	 
+	 std::vector<char> raw_in(0);
+	 if (action) {
+	   fseeko(f,0,SEEK_END);
+	   size = ftello(f);
+	   fseeko(f,0,SEEK_SET);
+
+	   raw_in.resize(size);
+	   assert(fread(&raw_in[0],size,1,f) == 1);
+	 }
+
+	 _grid->Barrier();
+	 gsw.Stop();
+
+	 RealD totalGB = (RealD)size / 1024./1024./1024 * _grid->_Nprocessors;
+	 RealD seconds = gsw.useconds() / 1e6;
+
+	 if (action) {
+	   std::cout << GridLogMessage << "[" << slot << "]  Read " << totalGB << " GB of compressed data at " << totalGB/seconds << " GB/s" << std::endl;
+	   
+	   uint32_t crc_comp = crc32_threaded((unsigned char*)&raw_in[0],size,0);
+	   
+	   if (crc_comp != crc32[slot]) {
+	     std::cout << "Node " << hostname << " found crc mismatch for file " << buf << " (" << std::hex << crc_comp << " vs " << crc32[slot] << std::dec << ")" << std::endl;
+	     std::cout << "Byte size: " << size << std::endl;
+	   }
+
+	   assert(crc_comp == crc32[slot]);
+	 }
+
+	 _grid->Barrier();
+       
+	 if (action) {
+	   fclose(f);
+	 }
+
+	 char* ptr = &raw_in[0];
+	 
+	 GridStopWatch gsw2;
+	 gsw2.Start();
+	 if (action) {
+	   int nsingleCap = nkeep_single;
+	   if (pr._evec.size() < nsingleCap)
+	     nsingleCap = pr._evec.size();
+	   
+	   int _cf_block_size = slot_lsites * 12 / 2 / blocks;
+	   
+#define FP_16_SIZE(a,b)  (( (a) + (a/b) )*2)
+	   
+	   // first read single precision basis vectors
+#pragma omp parallel
+	   {
+	     std::vector<float> buf(_cf_block_size * 2);
+#pragma omp for
+	     for (int nb=0;nb<blocks;nb++) {
+	       for (int i=0;i<nsingleCap;i++) {
+		 char* lptr = ptr + buf.size()*(i + nsingleCap*nb)*4;
+		 read_floats(lptr, &buf[0], buf.size() );
+		 int mnb = pr._bgrid.globalToLocalCanonicalBlock(slot,_nn,nb);
+		 if (mnb != -1)
+		   pr._bgrid.pokeBlockOfVectorCanonical(mnb,pr._evec._v[i],buf);
+	       }
+	     }
+	     
+#pragma omp barrier
+#pragma omp single
+	     {
+	       ptr = ptr + buf.size()*nsingleCap*blocks*4;
+	     }
+	     
+	   }
+	   
+	   // TODO: at this point I should add a checksum test for block_sp(nb,v,v) for all blocks, then I would know that the mapping
+	   // to blocks is OK at this point; after that ...
+	   
+	   // then read fixed precision basis vectors
+#pragma omp parallel
+	   {
+	     std::vector<float> buf(_cf_block_size * 2);
+#pragma omp for
+	     for (int nb=0;nb<blocks;nb++) {
+	       for (int i=nsingleCap;i<(int)pr._evec.size();i++) {
+		 char* lptr = ptr + FP_16_SIZE( buf.size(), 24 )*((i-nsingleCap) + (pr._evec.size() - nsingleCap)*nb);
+		 read_floats_fp16(lptr, &buf[0], buf.size(), 24);
+		 int mnb = pr._bgrid.globalToLocalCanonicalBlock(slot,_nn,nb);
+		 if (mnb != -1)
+		   pr._bgrid.pokeBlockOfVectorCanonical(mnb,pr._evec._v[i],buf);
+	       }
+	     }
+	     
+#pragma omp barrier
+#pragma omp single
+	     {
+	       ptr = ptr + FP_16_SIZE( buf.size()*(pr._evec.size() - nsingleCap)*blocks, 24 );
+	     }
+	   }
+	   
+#pragma omp parallel
+	   {
+	     std::vector<float> buf1(nkeep_single*2);
+	     std::vector<float> buf2((nkeep - nkeep_single)*2);
+	     
+#pragma omp for
+	     for (int j=0;j<(int)coef.size();j++)
+	       for (int nb=0;nb<blocks;nb++) {
+		 // get local coordinate on coarse grid
+		 int ii,oi;
+		 int mnb = pr._bgrid.globalToLocalCanonicalBlock(slot,_nn,nb);
+		 if (mnb != -1)
+		   canonical_block_to_coarse_coordinates(coef._v[0]._grid,mnb,ii,oi);
+		 
+		 char* lptr = ptr + (4*buf1.size() + FP_16_SIZE(buf2.size(), _FP16_COEF_EXP_SHARE_FLOATS))*(nb + j*blocks);
+		 int l;
+		 read_floats(lptr, &buf1[0], buf1.size() );
+		 if (mnb != -1) {
+		   for (l=0;l<nkeep_single;l++) {
+		     ((CoeffCoarse_t*)&coef._v[j]._odata[oi]._internal._internal[l])[ii] = CoeffCoarse_t(buf1[2*l+0],buf1[2*l+1]);
+		   }
+		 }
+		 read_floats_fp16(lptr, &buf2[0], buf2.size(), _FP16_COEF_EXP_SHARE_FLOATS);
+		 if (mnb != -1) {
+		   for (l=nkeep_single;l<nkeep;l++) {
+		     ((CoeffCoarse_t*)&coef._v[j]._odata[oi]._internal._internal[l])[ii] = CoeffCoarse_t(buf2[2*(l-nkeep_single)+0],buf2[2*(l-nkeep_single)+1]);
+		   }
+		 }
+		 
+	       }
+	   }
+	   
+	   // set checkerboard
+	   for (int i=0;i<(int)pr._evec.size();i++)
+	     pr._evec._v[i].checkerboard = Odd;
+	 
+	   gsw2.Stop();
+	   seconds=gsw2.useconds()/1e6;
+	   std::cout << GridLogMessage << "Processed " << totalGB << " GB of compressed data at " << totalGB/seconds << " GB/s" << std::endl;
+	 }
+       }
+     }
+#undef FP_16_SIZE
+     return true;
+
+   }
+ 
+   static bool DirectoryExists(const char *path) {
+     struct stat info;
+     return ((stat( path, &info ) == 0) && (info.st_mode & S_IFDIR));
+   }
+ 
+   static void conditionalMkDir(const char* path) {
+     if (!DirectoryExists(path))
+       mkdir(path,ACCESSPERMS);
+   }
+ 
+   template<typename Field,typename CoarseField>
+   static void write_compressed_vectors(const char* dir,const BlockProjector<Field>& pr,
+					const BasisFieldVector<CoarseField>& coef,
+					int nsingle,int writer_nodes = 0) {
+
+     GridStopWatch gsw;
+     
+     const BasisFieldVector<Field>& basis = pr._evec;
+     GridBase* _grid = basis._v[0]._grid;
+     std::vector<int> _l = _grid->FullDimensions();
+     for (int i=0;i<(int)_l.size();i++)
+       _l[i] /= _grid->_processors[i];
+
+     _grid->Barrier();
+     gsw.Start();
+
+     char buf[4096];
+     
+     // Making the directories is somewhat tricky.
+     // If we run on a joint filesystem we would just 
+     // have the boss create the directories and then
+     // have a barrier.  We also want to be able to run
+     // on local /scratch, so potentially all nodes need
+     // to create their own directories.  So do the following
+     // for now.
+     for (int j=0;j<_grid->_Nprocessors;j++) {
+       if (j == _grid->ThisRank()) {
+	 conditionalMkDir(dir);
+	 for (int i=0;i<32;i++) {
+	   sprintf(buf,"%s/%2.2d",dir,i);
+	   conditionalMkDir(buf);
+	 }       
+	 _grid->Barrier(); // make sure directories are ready
+       }
+     }
+     
+
+     typedef typename Field::scalar_type Coeff_t;
+     typedef typename CoarseField::scalar_type CoeffCoarse_t;
+     
+     int nperdir = _grid->_Nprocessors / 32;
+     if (nperdir < 1)
+       nperdir=1;
+
+     int slot;
+     Lexicographic::IndexFromCoor(_grid->_processor_coor,slot,_grid->_processors);
+
+     int64_t off = 0x0;
+     uint32_t crc = 0x0;
+     if (writer_nodes < 1)
+       writer_nodes = _grid->_Nprocessors;
+     int groups = _grid->_Nprocessors / writer_nodes;
+     if (groups<1)
+       groups = 1;
+
+     std::cout << GridLogMessage << " Write " << dir << " nodes = " << writer_nodes << std::endl;
+
+     for (int group=0;group<groups;group++) {
+       _grid->Barrier();
+       if (_grid->ThisRank() % groups == group) {
+	 
+	 sprintf(buf,"%s/%2.2d/%10.10d.compressed",dir,slot/nperdir,slot);
+	 FILE* f = fopen(buf,"wb");
+	 assert(f);
+
+	 //buffer does not seem to help
+	 //assert(!setvbuf ( f , NULL , _IOFBF , 1024*1024*2 ));
+	 	 
+	 int nsingleCap = nsingle;
+	 if (pr._evec.size() < nsingleCap)
+	   nsingleCap = pr._evec.size();
+	 
+	 GridStopWatch gsw1,gsw2,gsw3,gsw4,gsw5;
+
+	 gsw1.Start();
+
+	 std::vector<char> fbuf;
+	 fbuf.reserve( 1024 * 1024 * 8 );
+
+	 // first write single precision basis vectors
+	 for (int nb=0;nb<pr._bgrid._blocks;nb++) {
+	   for (int i=0;i<nsingleCap;i++) {
+	     std::vector<float> buf;
+	     pr._bgrid.peekBlockOfVectorCanonical(nb,pr._evec._v[i],buf);
+	     
+#if 0
+	     {
+	       RealD nrm = 0.0;
+	       for (int j=0;j<(int)buf.size();j++)
+		 nrm += buf[j]*buf[j];
+	       std::cout << GridLogMessage << "Norm: " << nrm << std::endl;
+	     }
+#endif
+	     write_floats(f,fbuf,crc, &buf[0], buf.size() );
+	   }
+	 }
+
+	 gsw1.Stop();
+	 gsw2.Start();
+
+	 // then write fixed precision basis vectors
+	 for (int nb=0;nb<pr._bgrid._blocks;nb++) {
+	   for (int i=nsingleCap;i<(int)pr._evec.size();i++) {
+	     std::vector<float> buf;
+	     pr._bgrid.peekBlockOfVectorCanonical(nb,pr._evec._v[i],buf);
+	     write_floats_fp16(f,fbuf,crc, &buf[0], buf.size(), 24);
+	   }
+	 }
+
+	 gsw2.Stop();
+	 assert(coef._v[0]._grid->_isites*coef._v[0]._grid->_osites == pr._bgrid._blocks);
+
+	 gsw3.Start();
+	 for (int j=0;j<(int)coef.size();j++) {
+
+	   int64_t size1 = nsingleCap*2;
+	   int64_t size2 = 2*(pr._evec.size()-nsingleCap);
+	   int64_t size = size1;
+	   if (size2>size)
+	     size=size2;
+	   std::vector<float> buf(size);
+
+	   //RealD nrmTest = 0.0;
+	   for (int nb=0;nb<pr._bgrid._blocks;nb++) {
+	     // get local coordinate on coarse grid
+	     int ii, oi;
+	     canonical_block_to_coarse_coordinates(coef._v[0]._grid,nb,ii,oi);
+	     
+	     gsw4.Start();
+	     gsw5.Start();
+	     for (int l=0;l<nsingleCap;l++) {
+	       auto res = ((CoeffCoarse_t*)&coef._v[j]._odata[oi]._internal._internal[l])[ii];
+	       buf[2*l+0] = res.real();
+	       buf[2*l+1] = res.imag();
+	       //nrmTest += res.real() * res.real() + res.imag() * res.imag();
+	     }
+	     gsw5.Stop();
+	     write_floats(f,fbuf,crc, &buf[0], size1 );
+	     gsw4.Stop();
+	     for (int l=nsingleCap;l<(int)pr._evec.size();l++) {
+	       auto res = ((CoeffCoarse_t*)&coef._v[j]._odata[oi]._internal._internal[l])[ii];
+	       buf[2*(l-nsingleCap)+0] = res.real();
+	       buf[2*(l-nsingleCap)+1] = res.imag();
+	       //nrmTest += res.real() * res.real() + res.imag() * res.imag();
+	     }
+	     write_floats_fp16(f,fbuf,crc, &buf[0], size2, FP16_COEF_EXP_SHARE_FLOATS);
+	   }
+
+	   //_grid->GlobalSum(nrmTest);
+	   //std::cout << GridLogMessage << "Test norm: " << nrmTest << std::endl;
+	 }
+	 gsw3.Stop();
+
+	 flush_bytes(f,fbuf);
+
+	 off = ftello(f);	 
+	 fclose(f);
+
+	 std::cout<<GridLogMessage << "Timing: write single basis in " << gsw1.Elapsed() << " and fp16 in " << gsw2.Elapsed() << " and coefficients in " << gsw3.Elapsed() << " (" <<
+	   gsw4.Elapsed() << ", " << gsw5.Elapsed() << ")" << std::endl;
+       }
+     }
+	 
+     _grid->Barrier();
+     gsw.Stop();
+     
+     RealD totalGB = (RealD)off / 1024./1024./1024 * _grid->_Nprocessors;
+     RealD seconds = gsw.useconds() / 1e6;
+     std::cout << GridLogMessage << "Write " << totalGB << " GB of compressed data at " << totalGB/seconds << " GB/s in " << seconds << " s" << std::endl;
+
+     // gather crcs
+     std::vector<uint32_t> crcs(_grid->_Nprocessors);
+     for (int i=0;i<_grid->_Nprocessors;i++) {
+       crcs[i] = 0x0;
+     }
+     crcs[slot] = crc;
+     for (int i=0;i<_grid->_Nprocessors;i++) {
+       _grid->GlobalSum(crcs[i]);
+     }
+     
+     if (_grid->IsBoss()) {
+       sprintf(buf,"%s/metadata.txt",dir);
+       FILE* f = fopen(buf,"wb");
+       assert(f);
+       for (int i=0;i<5;i++)
+	 fprintf(f,"s[%d] = %d\n",i,_grid->FullDimensions()[(i+1)%5] / _grid->_processors[(i+1)%5]);
+       for (int i=0;i<5;i++)
+	 fprintf(f,"b[%d] = %d\n",i,pr._bgrid._bs[(i+1)%5]);
+       for (int i=0;i<5;i++)
+	 fprintf(f,"nb[%d] = %d\n",i,pr._bgrid._nb[(i+1)%5]);
+       fprintf(f,"neig = %d\n",(int)coef.size());
+       fprintf(f,"nkeep = %d\n",(int)pr._evec.size());
+       fprintf(f,"nkeep_single = %d\n",nsingle);
+       fprintf(f,"blocks = %d\n",pr._bgrid._blocks);
+       fprintf(f,"FP16_COEF_EXP_SHARE_FLOATS = %d\n",FP16_COEF_EXP_SHARE_FLOATS);
+       for (int i =0;i<_grid->_Nprocessors;i++)
+	 fprintf(f,"crc32[%d] = %X\n",i,crcs[i]);
+       fclose(f);
+     }
+
+   }
+
+   template<typename Field>
+     static void write_argonne(const BasisFieldVector<Field>& ret,const char* dir) {
+     
+     GridBase* _grid = ret._v[0]._grid;
+     std::vector<int> _l = _grid->FullDimensions();
+     for (int i=0;i<(int)_l.size();i++)
+       _l[i] /= _grid->_processors[i];
+
+     char buf[4096];
+     
+     if (_grid->IsBoss()) {
+       mkdir(dir,ACCESSPERMS);
+       
+       for (int i=0;i<32;i++) {
+	 sprintf(buf,"%s/%2.2d",dir,i);
+	 mkdir(buf,ACCESSPERMS);
+       }
+     }
+     
+     _grid->Barrier(); // make sure directories are ready
+
+     
+     int nperdir = _grid->_Nprocessors / 32;
+     if (nperdir < 1)
+       nperdir=1;
+     std::cout << GridLogMessage << " Write " << dir << " nodes = " << _grid->_Nprocessors << std::endl;
+     
+     int slot;
+     Lexicographic::IndexFromCoor(_grid->_processor_coor,slot,_grid->_processors);
+     //printf("Slot: %d <> %d\n",slot, _grid->ThisRank());
+     
+     sprintf(buf,"%s/%2.2d/%10.10d",dir,slot/nperdir,slot);
+     FILE* f = fopen(buf,"wb");
+     assert(f);
+     
+     int N = (int)ret._v.size();
+     uint32_t crc = 0x0;
+     int64_t cf_size = _grid->oSites()*_grid->iSites()*12;
+     std::vector< float > rdata(cf_size*2);
+     
+     GridStopWatch gsw1,gsw2;
+     
+     for (int i=0;i<N;i++) {
+
+       if (ret._v[i].checkerboard != Odd)
+	 continue;
+
+       // create buffer and put data in argonne format in there
+       std::vector<int> coor(_l.size());
+       for (coor[1] = 0;coor[1]<_l[1];coor[1]++) {
+	 for (coor[2] = 0;coor[2]<_l[2];coor[2]++) {
+	   for (coor[3] = 0;coor[3]<_l[3];coor[3]++) {
+	     for (coor[4] = 0;coor[4]<_l[4];coor[4]++) {
+	       for (coor[0] = 0;coor[0]<_l[0];coor[0]++) {
+		 
+		 if ((coor[1]+coor[2]+coor[3]+coor[4]) % 2 == 1) {
+		   // peek
+		   iScalar<iVector<iVector<ComplexF, 3>, 4> > sc;
+		   peekLocalSite(sc,ret._v[i],coor);
+		   for (int s=0;s<4;s++)
+		     for (int c=0;c<3;c++)
+		       *(std::complex<float>*)&rdata[get_bfm_index(&coor[0],c+s*3, &_l[0] )] = sc()(s)(c);
+		 }
+	       }
+	     }
+	   }
+	 }
+       }
+       
+       // endian flip
+       for (int i=0;i<cf_size*2;i++) {
+	 char* c = (char*)&rdata[i];
+	 char tmp; int j;
+	 for (j=0;j<2;j++) {
+	   tmp = c[j]; c[j] = c[3-j]; c[3-j] = tmp;
+	 }
+       }
+       
+       // create crc of buffer
+       gsw1.Start();
+       crc = crc32_threaded((unsigned char*)&rdata[0],cf_size*2*4,crc);    
+       gsw1.Stop();
+       
+       // write out
+       gsw2.Start();
+       assert(fwrite(&rdata[0],cf_size*2*4,1,f)==1);
+       gsw2.Stop();
+       
+     }
+     
+     fclose(f);
+     
+     
+     // gather crc's and write out
+     std::vector<uint32_t> crcs(_grid->_Nprocessors);
+     for (int i=0;i<_grid->_Nprocessors;i++) {
+       crcs[i] = 0x0;
+     }
+     crcs[slot] = crc;
+     for (int i=0;i<_grid->_Nprocessors;i++) {
+       _grid->GlobalSum(crcs[i]);
+     }
+     
+     if (_grid->IsBoss()) {
+       sprintf(buf,"%s/checksums.txt",dir);
+       FILE* f = fopen(buf,"wt");
+       assert(f);
+       fprintf(f,"00000000\n\n");
+       for (int i =0;i<_grid->_Nprocessors;i++)
+	 fprintf(f,"%X\n",crcs[i]);
+       fclose(f);
+       
+       sprintf(buf,"%s/nodes.txt",dir);
+       f = fopen(buf,"wt");
+       assert(f);
+       for (int i =0;i<(int)_grid->_processors.size();i++)
+	 fprintf(f,"%d\n",_grid->_processors[i]);
+       fclose(f);
+     }
+     
+     
+     std::cout << GridLogMessage << "Writing slot " << slot << " with "
+	       << N << " vectors in "
+	       << gsw2.Elapsed() << " at " 
+	       << ( (double)cf_size*2*4 * N / 1024./1024./1024. / gsw2.useconds()*1000.*1000. )
+	       << " GB/s  with crc computed at "
+	       << ( (double)cf_size*2*4 * N / 1024./1024./1024. / gsw1.useconds()*1000.*1000. )
+	       << " GB/s "
+	       << std::endl;
+     
+     _grid->Barrier();
+     std::cout << GridLogMessage  << "Writing complete" << std::endl;
+     
+   }
+ }
+
+}
diff --git a/lib/qcd/action/fermion/DomainWallEOFAFermion.cc b/lib/qcd/action/fermion/DomainWallEOFAFermion.cc
index dd8a500d..37ab5fa6 100644
--- a/lib/qcd/action/fermion/DomainWallEOFAFermion.cc
+++ b/lib/qcd/action/fermion/DomainWallEOFAFermion.cc
@@ -61,10 +61,10 @@ namespace QCD {
     }
 
     /***************************************************************
-    /* Additional EOFA operators only called outside the inverter.
-    /* Since speed is not essential, simple axpby-style
-    /* implementations should be fine.
-    /***************************************************************/
+     * Additional EOFA operators only called outside the inverter.
+     * Since speed is not essential, simple axpby-style
+     * implementations should be fine.
+     ***************************************************************/
     template<class Impl>
     void DomainWallEOFAFermion<Impl>::Omega(const FermionField& psi, FermionField& Din, int sign, int dag)
     {
@@ -116,8 +116,8 @@ namespace QCD {
     }
 
     /********************************************************************
-    /* Performance critical fermion operators called inside the inverter
-    /********************************************************************/
+     * Performance critical fermion operators called inside the inverter
+     ********************************************************************/
 
     template<class Impl>
     void DomainWallEOFAFermion<Impl>::M5D(const FermionField& psi, FermionField& chi)
diff --git a/lib/qcd/action/fermion/MobiusEOFAFermion.cc b/lib/qcd/action/fermion/MobiusEOFAFermion.cc
index 085fa988..0344afbf 100644
--- a/lib/qcd/action/fermion/MobiusEOFAFermion.cc
+++ b/lib/qcd/action/fermion/MobiusEOFAFermion.cc
@@ -77,11 +77,11 @@ namespace QCD {
       }
     }
 
-    /***************************************************************
-    /* Additional EOFA operators only called outside the inverter.
-    /* Since speed is not essential, simple axpby-style
-    /* implementations should be fine.
-    /***************************************************************/
+    /****************************************************************
+     * Additional EOFA operators only called outside the inverter.  
+     * Since speed is not essential, simple axpby-style
+     * implementations should be fine.
+     ***************************************************************/
     template<class Impl>
     void MobiusEOFAFermion<Impl>::Omega(const FermionField& psi, FermionField& Din, int sign, int dag)
     {
@@ -194,8 +194,8 @@ namespace QCD {
     }
 
     /********************************************************************
-    /* Performance critical fermion operators called inside the inverter
-    /********************************************************************/
+     * Performance critical fermion operators called inside the inverter
+     ********************************************************************/
 
     template<class Impl>
     void MobiusEOFAFermion<Impl>::M5D(const FermionField& psi, FermionField& chi)
diff --git a/tests/solver/Params.h b/tests/solver/Params.h
new file mode 100644
index 00000000..d9a6d3b3
--- /dev/null
+++ b/tests/solver/Params.h
@@ -0,0 +1,136 @@
+/*
+  Params IO
+
+  Author: Christoph Lehner
+  Date:   2017
+*/
+
+#define PADD(p,X) p.get(#X,X);
+
+class Params {
+ protected:
+
+  std::string trim(const std::string& sc) {
+    std::string s = sc;
+    s.erase(s.begin(), std::find_if(s.begin(), s.end(),
+				    std::not1(std::ptr_fun<int, int>(std::isspace))));
+    s.erase(std::find_if(s.rbegin(), s.rend(),
+			 std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end());
+    return s;
+  }
+
+ public:
+
+  std::map< std::string, std::string > lines;
+  std::string _fn;
+
+ Params(const char* fn) : _fn(fn) {
+    FILE* f = fopen(fn,"rt");
+    assert(f);
+    while (!feof(f)) {
+      char buf[4096];
+      if (fgets(buf,sizeof(buf),f)) {
+	if (buf[0] != '#' && buf[0] != '\r' && buf[0] != '\n') {
+	  char* sep = strchr(buf,'=');
+	  assert(sep);
+	  *sep = '\0';
+	  lines[trim(buf)] = trim(sep+1);
+	}
+      }
+    }      
+    fclose(f);
+  }
+
+  ~Params() {
+  }
+
+  std::string loghead() {
+    return _fn + ": ";
+  }
+
+  bool has(const char* name) {
+    auto f = lines.find(name);
+    return (f != lines.end());
+  }
+
+  const std::string& get(const char* name) {
+    auto f = lines.find(name);
+    if (f == lines.end()) {
+      std::cout << Grid::GridLogMessage << loghead() << "Could not find value for " << name << std::endl;
+      abort();
+    }
+    return f->second;
+  }
+
+  void parse(std::string& s, const std::string& cval) {
+    std::stringstream trimmer;
+    trimmer << cval;
+    s.clear();
+    trimmer >> s;
+  }
+
+  void parse(int& i, const std::string& cval) {
+    assert(sscanf(cval.c_str(),"%d",&i)==1);
+  }
+
+  void parse(long long& i, const std::string& cval) {
+    assert(sscanf(cval.c_str(),"%lld",&i)==1);
+  }
+
+  void parse(double& f, const std::string& cval) {
+    assert(sscanf(cval.c_str(),"%lf",&f)==1);
+  }
+
+  void parse(float& f, const std::string& cval) {
+    assert(sscanf(cval.c_str(),"%f",&f)==1);
+  }
+
+  void parse(bool& b, const std::string& cval) {
+    std::string lcval = cval;
+    std::transform(lcval.begin(), lcval.end(), lcval.begin(), ::tolower);
+    if (lcval == "true" || lcval == "yes") {
+      b = true;
+    } else if (lcval == "false" || lcval == "no") {
+      b = false;
+    } else {
+      std::cout << "Invalid value for boolean: " << b << std::endl;
+      assert(0);
+    }
+  }
+
+  void parse(std::complex<double>& f, const std::string& cval) {
+    double r,i;
+    assert(sscanf(cval.c_str(),"%lf %lf",&r,&i)==2);
+    f = std::complex<double>(r,i);
+  }
+
+  void parse(std::complex<float>& f, const std::string& cval) {
+    float r,i;
+    assert(sscanf(cval.c_str(),"%f %f",&r,&i)==2);
+    f = std::complex<float>(r,i);
+  }
+
+  template<class T>
+    void get(const char* name, std::vector<T>& v) {
+    int i = 0;
+    v.resize(0);
+    while (true) {
+      char buf[4096];
+      sprintf(buf,"%s[%d]",name,i++);
+      if (!has(buf))
+	break;
+      T val;
+      parse(val,get(buf));
+      std::cout << Grid::GridLogMessage << loghead() << "Set " << buf << " to " << val << std::endl;
+      v.push_back(val);
+    }
+  }
+
+  template<class T>
+    void get(const char* name, T& f) {
+    parse(f,get(name));
+    std::cout << Grid::GridLogMessage << loghead() << "Set " << name << " to " << f << std::endl;
+  }
+
+  
+};
diff --git a/tests/solver/Test_dwf_compressed_lanczos.cc b/tests/solver/Test_dwf_compressed_lanczos.cc
new file mode 100644
index 00000000..b42a2d55
--- /dev/null
+++ b/tests/solver/Test_dwf_compressed_lanczos.cc
@@ -0,0 +1,727 @@
+/*
+  Authors: Christoph Lehner
+  Date: 2017
+
+  Multigrid Lanczos
+
+
+
+  TODO:
+
+  High priority:
+  - Explore filtering of starting vector again, should really work:  If cheby has 4 for low mode region and 1 for high mode, applying 15 iterations has 1e9 suppression
+    of high modes, which should create the desired invariant subspace already?  Missing something here???  Maybe dynamic range dangerous, i.e., could also kill interesting
+    eigenrange if not careful.
+
+    Better: Use all Cheby up to order N in order to approximate a step function; try this!  Problem: width of step function.  Can kill eigenspace > 1e-3 and have < 1e-5 equal
+            to 1
+
+  Low priority:
+  - Given that I seem to need many restarts and high degree poly to create the base and this takes about 1 day, seriously consider a simple method to create a basis
+    (ortho krylov low poly); and then fix up lowest say 200 eigenvalues by 1 run with high-degree poly (600 could be enough)
+*/
+#include <Grid/Grid.h>
+#include "Params.h"
+
+#include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h>
+
+using namespace std;
+using namespace Grid;
+using namespace Grid::QCD;
+
+bool read_evals(GridBase* _grid, char* fn, std::vector<RealD>& evals) {
+
+  FILE* f = 0;
+  uint32_t status = 0;
+  if (_grid->IsBoss()) {
+    f = fopen(fn,"rt");
+    status = f ? 1 : 0;
+  }
+  _grid->GlobalSum(status);
+
+  if (!status)
+    return false;
+
+  uint32_t N;
+  if (f)
+    assert(fscanf(f,"%d\n",&N)==1);
+  else
+    N = 0;
+  _grid->GlobalSum(N);
+
+  std::cout << "Reading " << N << " eigenvalues" << std::endl;
+
+  evals.resize(N);
+
+  for (int i=0;i<N;i++) {
+    if (f)
+      assert(fscanf(f,"%lf",&evals[i])==1);
+    else
+      evals[i] = 0;
+  }
+
+  _grid->GlobalSumVector(&evals[0],evals.size());
+
+  if (f)
+    fclose(f);
+  return true;
+}
+
+void write_evals(char* fn, std::vector<RealD>& evals) {
+  FILE* f = fopen(fn,"wt");
+  assert(f);
+
+  int N = (int)evals.size();
+  fprintf(f,"%d\n",N);
+
+  for (int i=0;i<N;i++) {
+    fprintf(f,"%.15E\n",evals[i]);
+  }
+
+  fclose(f);
+}
+
+void write_history(char* fn, std::vector<RealD>& hist) {
+  FILE* f = fopen(fn,"wt");
+  assert(f);
+
+  int N = (int)hist.size();
+  for (int i=0;i<N;i++) {
+    fprintf(f,"%d %.15E\n",i,hist[i]);
+  }
+
+  fclose(f);
+}
+
+template<typename Field>
+class FunctionHermOp : public LinearFunction<Field> {
+public:
+  OperatorFunction<Field>   & _poly;
+  LinearOperatorBase<Field> &_Linop;
+
+  FunctionHermOp(OperatorFunction<Field> & poly,LinearOperatorBase<Field>& linop) : _poly(poly), _Linop(linop) {
+  }
+
+  void operator()(const Field& in, Field& out) {
+    _poly(_Linop,in,out);
+  }
+};
+
+template<typename Field>
+class CheckpointedLinearFunction : public LinearFunction<Field> {
+public:
+  LinearFunction<Field>& _op;
+  std::string _dir;
+  int _max_apply;
+  int _apply, _apply_actual;
+  GridBase* _grid;
+  FILE* _f;
+
+  CheckpointedLinearFunction(GridBase* grid, LinearFunction<Field>& op, const char* dir,int max_apply) : _op(op), _dir(dir), _grid(grid), _f(0),
+													 _max_apply(max_apply), _apply(0), _apply_actual(0) {
+
+    FieldVectorIO::conditionalMkDir(dir);
+
+    char fn[4096];
+    sprintf(fn,"%s/ckpt_op.%4.4d",_dir.c_str(),_grid->ThisRank());
+    printf("CheckpointLinearFunction:: file %s\n",fn);
+    _f = fopen(fn,"r+b");
+    if (!_f)
+      _f = fopen(fn,"w+b");
+    assert(_f);
+    fseek(_f,0,SEEK_CUR);
+
+  }
+
+  ~CheckpointedLinearFunction() {
+    if (_f) {
+      fclose(_f);
+      _f = 0;
+    }
+  }
+
+  bool load_ckpt(const Field& in, Field& out) {
+
+    off_t cur = ftello(_f);
+    fseeko(_f,0,SEEK_END);
+    if (cur == ftello(_f))
+      return false;
+    fseeko(_f,cur,SEEK_SET);
+
+    size_t sz = sizeof(out._odata[0]) * out._odata.size();
+
+    GridStopWatch gsw;
+    gsw.Start();
+    uint32_t crc_exp;
+    assert(fread(&crc_exp,4,1,_f)==1);
+    assert(fread(&out._odata[0],sz,1,_f)==1);
+    assert(FieldVectorIO::crc32_threaded((unsigned char*)&out._odata[0],sz,0x0)==crc_exp);
+    gsw.Stop();
+
+    printf("CheckpointLinearFunction:: reading %lld\n",(long long)sz);
+    std::cout << GridLogMessage << "Loading " << ((RealD)sz/1024./1024./1024.) << " GB in " << gsw.Elapsed() << std::endl;
+    return true;
+  }
+
+  void save_ckpt(const Field& in, Field& out) {
+
+    fseek(_f,0,SEEK_CUR); // switch to write
+
+    size_t sz = sizeof(out._odata[0]) * out._odata.size();
+
+    GridStopWatch gsw;
+    gsw.Start();
+    uint32_t crc = FieldVectorIO::crc32_threaded((unsigned char*)&out._odata[0],sz,0x0);
+    assert(fwrite(&crc,4,1,_f)==1);
+    assert(fwrite(&out._odata[0],sz,1,_f)==1);
+    fflush(_f); // try this on the GPFS to suppress OPA usage for disk during dslash; this is not needed at Lustre/JLAB
+    gsw.Stop();
+
+    printf("CheckpointLinearFunction:: writing %lld\n",(long long)sz);
+    std::cout << GridLogMessage << "Saving " << ((RealD)sz/1024./1024./1024.) << " GB in " << gsw.Elapsed() << std::endl;
+  }
+
+  void operator()(const Field& in, Field& out) {
+
+    _apply++;
+
+    if (load_ckpt(in,out))
+      return;
+
+    _op(in,out);
+    
+    save_ckpt(in,out);
+
+    if (_apply_actual++ >= _max_apply) {
+      std::cout << GridLogMessage << "Maximum application of operator reached, checkpoint and finish in future job" << std::endl;
+      if (_f) { fclose(_f); _f=0; }
+      in._grid->Barrier();
+      Grid_finalize();
+      exit(3);
+    }
+  }
+};
+
+template<typename CoarseField,typename Field>
+class ProjectedFunctionHermOp : public LinearFunction<CoarseField> {
+public:
+  OperatorFunction<Field>   & _poly;
+  LinearOperatorBase<Field> &_Linop;
+  BlockProjector<Field>& _pr;
+
+  ProjectedFunctionHermOp(BlockProjector<Field>& pr,OperatorFunction<Field> & poly,LinearOperatorBase<Field>& linop) : _poly(poly), _Linop(linop), _pr(pr) {
+  }
+
+  void operator()(const CoarseField& in, CoarseField& out) {
+    assert(_pr._bgrid._o_blocks == in._grid->oSites());
+
+    Field fin(_pr._bgrid._grid);
+    Field fout(_pr._bgrid._grid);
+
+    GridStopWatch gsw1,gsw2,gsw3;
+    // fill fin
+    gsw1.Start();
+    _pr.coarseToFine(in,fin);
+    gsw1.Stop();
+
+    // apply poly
+    gsw2.Start();
+    _poly(_Linop,fin,fout);
+    gsw2.Stop();
+
+    // fill out
+    gsw3.Start();
+    _pr.fineToCoarse(fout,out);
+    gsw3.Stop();
+
+    auto eps = innerProduct(in,out);
+    std::cout << GridLogMessage << "Operator timing details: c2f = " << gsw1.Elapsed() << " poly = " << gsw2.Elapsed() << " f2c = " << gsw3.Elapsed() << 
+      "   Complimentary Hermiticity check: " << eps.imag() / std::abs(eps) << std::endl;
+
+  }
+};
+
+template<typename CoarseField,typename Field>
+class ProjectedHermOp : public LinearFunction<CoarseField> {
+public:
+  LinearOperatorBase<Field> &_Linop;
+  BlockProjector<Field>& _pr;
+
+  ProjectedHermOp(BlockProjector<Field>& pr,LinearOperatorBase<Field>& linop) : _Linop(linop), _pr(pr) {
+  }
+
+  void operator()(const CoarseField& in, CoarseField& out) {
+    assert(_pr._bgrid._o_blocks == in._grid->oSites());
+    Field fin(_pr._bgrid._grid);
+    Field fout(_pr._bgrid._grid);
+    _pr.coarseToFine(in,fin);
+    _Linop.HermOp(fin,fout);
+    _pr.fineToCoarse(fout,out);
+
+  }
+};
+
+template<typename Field>
+class PlainHermOp : public LinearFunction<Field> {
+public:
+  LinearOperatorBase<Field> &_Linop;
+
+  PlainHermOp(LinearOperatorBase<Field>& linop) : _Linop(linop) {
+  }
+
+  void operator()(const Field& in, Field& out) {
+    _Linop.HermOp(in,out);
+  }
+};
+
+template<typename vtype, int N > using CoarseSiteFieldGeneral = iScalar< iVector<vtype, N> >;
+template<int N> using CoarseSiteFieldD = CoarseSiteFieldGeneral< vComplexD, N >;
+template<int N> using CoarseSiteFieldF = CoarseSiteFieldGeneral< vComplexF, N >;
+template<int N> using CoarseSiteField  = CoarseSiteFieldGeneral< vComplex,  N >;
+template<int N> using CoarseLatticeFermion  = Lattice< CoarseSiteField<N> >;
+template<int N> using CoarseLatticeFermionD = Lattice< CoarseSiteFieldD<N> >;
+
+template<typename Field,int Nstop1>
+void CoarseGridLanczos(BlockProjector<Field>& pr,RealD alpha2,RealD beta,int Npoly2,
+		       int Nstop2,int Nk2,int Nm2,RealD resid2,RealD betastp2,int MaxIt,int MinRes2,
+		       LinearOperatorBase<Field>& HermOp, std::vector<RealD>& eval1, bool cg_test_enabled, 
+		       int cg_test_maxiter,int nsingle,int SkipTest2, int MaxApply2,bool smoothed_eval_enabled,
+		       int smoothed_eval_inner,int smoothed_eval_outer,int smoothed_eval_begin,
+		       int smoothed_eval_end,RealD smoothed_eval_inner_resid) {
+
+  BlockedGrid<Field>& bgrid = pr._bgrid;
+  BasisFieldVector<Field>& basis = pr._evec;
+
+
+  std::vector<int> coarseFourDimLatt;
+  for (int i=0;i<4;i++)
+    coarseFourDimLatt.push_back(bgrid._nb[1+i] * bgrid._grid->_processors[1+i]);
+  assert(bgrid._grid->_processors[0] == 1);
+
+  std::cout << GridLogMessage << "CoarseGrid = " << coarseFourDimLatt << " with basis = " << Nstop1 << std::endl;
+  GridCartesian         * UCoarseGrid   = SpaceTimeGrid::makeFourDimGrid(coarseFourDimLatt, GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridCartesian         * FCoarseGrid   = SpaceTimeGrid::makeFiveDimGrid(bgrid._nb[0],UCoarseGrid);
+
+  Chebyshev<Field> Cheb2(alpha2,beta,Npoly2);
+  CoarseLatticeFermion<Nstop1> src_coarse(FCoarseGrid);
+
+  // Second round of Lanczos in blocked space
+  std::vector<RealD>         eval2(Nm2);
+  std::vector<RealD>         eval3(Nm2);
+  BasisFieldVector<CoarseLatticeFermion<Nstop1> > coef(Nm2,FCoarseGrid);
+
+  ProjectedFunctionHermOp<CoarseLatticeFermion<Nstop1>,LatticeFermion> Op2plain(pr,Cheb2,HermOp);
+  CheckpointedLinearFunction<CoarseLatticeFermion<Nstop1> > Op2ckpt(src_coarse._grid,Op2plain,"checkpoint",MaxApply2);
+  LinearFunction< CoarseLatticeFermion<Nstop1> >* Op2;
+  if (MaxApply2) {
+    Op2 = &Op2ckpt;
+  } else {
+    Op2 = &Op2plain;
+  }
+  ProjectedHermOp<CoarseLatticeFermion<Nstop1>,LatticeFermion> Op2nopoly(pr,HermOp);
+  BlockImplicitlyRestartedLanczos<CoarseLatticeFermion<Nstop1> > IRL2(*Op2,*Op2,Nstop2,Nk2,Nm2,resid2,betastp2,MaxIt,MinRes2);
+
+
+  src_coarse = 1.0;
+  
+  // Precision test
+  {
+    Field tmp(bgrid._grid);
+    CoarseLatticeFermion<Nstop1> tmp2(FCoarseGrid);
+    CoarseLatticeFermion<Nstop1> tmp3(FCoarseGrid);
+    tmp2 = 1.0;
+    tmp3 = 1.0;
+
+    pr.coarseToFine(tmp2,tmp);
+    pr.fineToCoarse(tmp,tmp2);
+
+    tmp2 -= tmp3;
+    std::cout << GridLogMessage << "Precision Test c->f->c: " << norm2(tmp2) / norm2(tmp3) << std::endl;
+
+    //bgrid._grid->Barrier();
+    //return;
+  }
+
+  int Nconv;
+  if (!FieldVectorIO::read_compressed_vectors("lanczos.output",pr,coef) ||
+      !read_evals(UCoarseGrid,(char *)"lanczos.output/eigen-values.txt",eval3) ||
+      !read_evals(UCoarseGrid,(char *)"lanczos.output/eigen-values.txt.linear",eval1) ||
+      !read_evals(UCoarseGrid,(char *)"lanczos.output/eigen-values.txt.poly",eval2)
+      ) {
+    
+
+    IRL2.calc(eval2,coef,src_coarse,Nconv,true,SkipTest2);
+
+    coef.resize(Nstop2);
+    eval2.resize(Nstop2);
+    eval3.resize(Nstop2);
+
+    std::vector<Field> step3_cache;
+
+    // reconstruct eigenvalues of original operator
+    for (int i=0;i<Nstop2;i++){
+      RealD eval2_linear;
+
+      if (i<Nstop1) {
+	eval2_linear = eval1[i];
+      } else {
+	eval2_linear = eval2[i-1];
+      }
+
+      RealD eval2_poly = eval2[i];
+      RealD eval_reconstruct = Cheb2.approxInv(eval2_poly,eval2_linear,100,1e-10);
+      std::cout << i << " Reconstructed eval = " << eval_reconstruct << " from quess " << eval2_linear << std::endl;
+      eval2[i] = eval_reconstruct;
+    }
+    
+    // as demonstrated in CG test below, best result from mixed determination
+    for (int i=0;i<Nstop2;i++)
+      eval3[i] = (i < Nstop1) ? eval1[i] : eval2[i];
+    
+    for(int i=0;i<Nstop2;i++){
+      std::cout << i<<" / "<< Nstop2<< " eigenvalue "<< eval3[i] <<std::endl;
+    };
+    
+    // write
+    mkdir("lanczos.output",ACCESSPERMS);
+    FieldVectorIO::write_compressed_vectors("lanczos.output",pr,coef,nsingle);
+    if (bgrid._grid->IsBoss()) {
+      write_evals((char *)"lanczos.output/eigen-values.txt",eval3);
+      write_evals((char *)"lanczos.output/eigen-values.txt.linear",eval1);
+      write_evals((char *)"lanczos.output/eigen-values.txt.poly",eval2);
+    }
+
+  }
+
+  // fix up eigenvalues
+  if (!read_evals(UCoarseGrid,(char *)"lanczos.output/eigen-values.txt.smoothed",eval3) && smoothed_eval_enabled) {
+
+    ConjugateGradient<LatticeFermion> CG(smoothed_eval_inner_resid, smoothed_eval_inner, false);
+
+    LatticeFermion v_i(basis[0]._grid);
+    auto tmp = v_i;
+    auto tmp2 = v_i;
+
+    for (int i=smoothed_eval_begin;i<smoothed_eval_end;i++) {
+
+      GridStopWatch gsw;
+
+      gsw.Start();
+
+      pr.coarseToFine(coef[i],v_i);
+      v_i.checkerboard = Odd;
+      
+      for (int j=0;j<smoothed_eval_outer;j++) {
+	tmp=zero;
+	//pr.deflate(coef,eval3,Nstop2,v_i,tmp);
+	CG(HermOp, v_i, tmp);
+
+	v_i = 1.0 / ::sqrt( norm2(tmp) ) * tmp;
+      }
+
+      tmp = v_i;
+
+      HermOp.HermOp(tmp,tmp2);
+
+      RealD ev = innerProduct(tmp,tmp2).real();
+
+      gsw.Stop();
+
+      std::cout << GridLogMessage << "Smoothed eigenvalue " << i << " from " << eval3[i] << " to " << ev << " in " << gsw.Elapsed() << std::endl;
+      //	" with effective smoother precision " << (CG.ResHistory.back() / CG.ResHistory.front() ) << std::endl;
+      //      CG.ResHistory.clear();
+
+      eval3[i] = ev;
+    }
+
+    if (bgrid._grid->IsBoss()) {
+      write_evals((char *)"lanczos.output/eigen-values.txt.smoothed",eval3);
+      write_evals((char *)"lanczos.output/eigen-values.txt",eval3); // also reset this to the best ones we have available
+    }
+  }
+
+  // do CG test with and without deflation
+  if (cg_test_enabled) {
+    ConjugateGradient<LatticeFermion> CG(1.0e-8, cg_test_maxiter, false);
+    LatticeFermion src_orig(bgrid._grid);
+    src_orig.checkerboard = Odd;
+    src_orig = 1.0;
+    src_orig = src_orig * (1.0 / ::sqrt(norm2(src_orig)) );
+    auto result = src_orig; 
+
+    // undeflated solve
+    result = zero;
+    CG(HermOp, src_orig, result);
+    //    if (UCoarseGrid->IsBoss())
+    //      write_history("cg_test.undefl",CG.ResHistory);
+    //    CG.ResHistory.clear();
+
+    // deflated solve with all eigenvectors
+    result = zero;
+    pr.deflate(coef,eval2,Nstop2,src_orig,result);
+    CG(HermOp, src_orig, result);
+    //    if (UCoarseGrid->IsBoss())
+    //      write_history("cg_test.defl_all",CG.ResHistory);
+    //    CG.ResHistory.clear();
+
+    // deflated solve with non-blocked eigenvectors
+    result = zero;
+    pr.deflate(coef,eval1,Nstop1,src_orig,result);
+    CG(HermOp, src_orig, result);
+    //    if (UCoarseGrid->IsBoss())
+    //      write_history("cg_test.defl_full",CG.ResHistory);
+    //    CG.ResHistory.clear();
+
+    // deflated solve with all eigenvectors and original eigenvalues from proj
+    result = zero;
+    pr.deflate(coef,eval3,Nstop2,src_orig,result);
+    CG(HermOp, src_orig, result);
+    //    if (UCoarseGrid->IsBoss())
+    //      write_history("cg_test.defl_all_ev3",CG.ResHistory);
+    //    CG.ResHistory.clear();
+
+  }
+  
+}
+
+
+template<typename Field>
+void quick_krylov_basis(BasisFieldVector<Field>& evec,Field& src,LinearFunction<Field>& Op,int Nstop) {
+  Field tmp = src;
+  Field tmp2 = tmp;
+
+  for (int i=0;i<Nstop;i++) {
+    GridStopWatch gsw;
+    gsw.Start();
+    Op(tmp,tmp2);
+    gsw.Stop();
+    evec.orthogonalize(tmp2,i);
+
+    RealD nn = norm2(tmp2);
+    nn = Grid::sqrt(nn);
+    tmp2 = tmp2 * (1.0/nn);
+
+    evec[i] = tmp2;
+    tmp = tmp2;
+    std::cout << GridLogMessage << "Quick_krylov_basis: " << i << "/" << Nstop << " timing of operator=" << gsw.Elapsed() << std::endl;
+  }
+
+}
+
+
+
+int main (int argc, char ** argv) {
+
+  Grid_init(&argc,&argv);
+
+  const int MaxIt = 10000;
+
+  int Ls;
+  RealD mass;
+  RealD M5;
+  std::vector < std::complex<double>  > omega;
+  
+  RealD alpha1, alpha2, beta;
+  int Npoly1, Npoly2;
+  int Nstop1, Nstop2;
+  int Nk1, Nk2;
+  int Np1, Np2;
+  int MinRes1, MinRes2;
+  int SkipTest2, MaxApply2;
+  bool checkpoint_basis;
+  bool cg_test_enabled;
+  bool exit_after_basis_calculation;
+  bool simple_krylov_basis;
+  int cg_test_maxiter;
+  int nsingle; // store in single precision, the rest in FP16
+  int max_cheb_time_ms;
+  bool smoothed_eval_enabled;
+  int smoothed_eval_inner;
+  int smoothed_eval_outer;
+  int smoothed_eval_begin;
+  int smoothed_eval_end;
+  RealD smoothed_eval_inner_resid;
+
+  // vector representation
+  std::vector<int> block_size; // 5d block size
+
+  RealD resid1, resid2, betastp1, betastp2, basis_norm_threshold;
+
+  std::string config;
+  
+  Params jp("params.txt");
+  PADD(jp,Npoly1); PADD(jp,Npoly2);
+  PADD(jp,max_cheb_time_ms);
+  PADD(jp,Nstop1); PADD(jp,Nstop2); PADD(jp,MaxApply2);
+  PADD(jp,Nk1); PADD(jp,Nk2); PADD(jp,betastp1); PADD(jp,betastp2);
+  PADD(jp,Np1); PADD(jp,Np2); basis_norm_threshold = 1e-5; //PADD(jp,basis_norm_threshold);
+  PADD(jp,block_size); PADD(jp,smoothed_eval_enabled); PADD(jp,smoothed_eval_inner);
+  PADD(jp,resid1); PADD(jp,resid2); PADD(jp,smoothed_eval_outer);
+  PADD(jp,alpha1); PADD(jp,alpha2); PADD(jp,smoothed_eval_begin);
+  PADD(jp,MinRes1); PADD(jp,MinRes2); PADD(jp,smoothed_eval_end);
+  PADD(jp,beta); PADD(jp,mass); PADD(jp,smoothed_eval_inner_resid);
+  PADD(jp,omega); PADD(jp,config); 
+  PADD(jp,M5); PADD(jp,cg_test_enabled);
+  PADD(jp,cg_test_maxiter); PADD(jp,checkpoint_basis);
+  PADD(jp,nsingle); PADD(jp,exit_after_basis_calculation);
+  PADD(jp,simple_krylov_basis); PADD(jp,SkipTest2);
+
+  Ls = (int)omega.size();
+
+  // Grids
+  GridCartesian         * UGrid   = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplex::Nsimd()),GridDefaultMpi());
+  GridCartesian         * UGridHP = SpaceTimeGrid::makeFourDimGrid(GridDefaultLatt(), GridDefaultSimd(Nd,vComplexD::Nsimd()),GridDefaultMpi());
+  GridRedBlackCartesian * UrbGrid = SpaceTimeGrid::makeFourDimRedBlackGrid(UGrid);
+  GridRedBlackCartesian * UrbGridHP = SpaceTimeGrid::makeFourDimRedBlackGrid(UGridHP);
+  GridCartesian         * FGrid   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGrid);
+  GridCartesian         * FGridHP   = SpaceTimeGrid::makeFiveDimGrid(Ls,UGridHP);
+  GridRedBlackCartesian * FrbGrid = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGrid);
+  GridRedBlackCartesian * FrbGridHP = SpaceTimeGrid::makeFiveDimRedBlackGrid(Ls,UGridHP);
+
+  // Gauge field
+  LatticeGaugeField Umu(UGrid);
+  FieldMetaData header;
+  NerscIO::readConfiguration(Umu,header,config);
+  std::cout << GridLogMessage << "Lattice dimensions: " << GridDefaultLatt()
+            << "   Ls: " << Ls << std::endl;
+
+  // ZMobius EO Operator
+  ZMobiusFermionR Ddwf(Umu, *FGrid, *FrbGrid, *UGrid, *UrbGrid, mass, M5, omega,1.,0.);
+  SchurDiagTwoOperator<ZMobiusFermionR,LatticeFermion> HermOp(Ddwf);
+
+  // Eigenvector storage
+  const int Nm1 = Np1 + Nk1;
+  const int Nm2 = Np2 + Nk2; // maximum number of vectors we need to keep
+  std::cout << GridLogMessage << "Keep " << Nm1 << " full vectors" << std::endl;
+  std::cout << GridLogMessage << "Keep " << Nm2 << " total vectors" << std::endl;
+  assert(Nm2 >= Nm1);
+  BasisFieldVector<LatticeFermion> evec(Nm1,FrbGrid); // start off with keeping full vectors
+
+  // First and second cheby
+  Chebyshev<LatticeFermion> Cheb1(alpha1,beta,Npoly1);
+  FunctionHermOp<LatticeFermion> Op1(Cheb1,HermOp);
+  PlainHermOp<LatticeFermion> Op1test(HermOp);
+
+  // Eigenvalue storage
+  std::vector<RealD>          eval1(evec.size());
+
+  // Construct source vector
+  LatticeFermion    src(FrbGrid);
+  {
+    src=1.0;
+    src.checkerboard = Odd;
+
+    // normalize
+    RealD nn = norm2(src);
+    nn = Grid::sqrt(nn);
+    src = src * (1.0/nn);
+  }
+
+  // Do a benchmark and a quick exit if performance is too little (ugly but needed due to performance fluctuations)
+  if (max_cheb_time_ms) {
+    // one round of warmup
+    auto tmp = src;
+    GridStopWatch gsw1,gsw2;
+    gsw1.Start();
+    Cheb1(HermOp,src,tmp);
+    gsw1.Stop();
+    Ddwf.ZeroCounters();
+    gsw2.Start();
+    Cheb1(HermOp,src,tmp);
+    gsw2.Stop();
+    Ddwf.Report();
+    std::cout << GridLogMessage << "Performance check; warmup = " << gsw1.Elapsed() << "  test = " << gsw2.Elapsed() << std::endl;
+    int ms = (int)(gsw2.useconds()/1e3);
+    if (ms > max_cheb_time_ms) {
+      std::cout << GridLogMessage << "Performance too poor: " << ms << " ms, cutoff = " << max_cheb_time_ms << " ms" << std::endl;
+      Grid_finalize();
+      return 2;
+    }
+
+  }
+
+  // First round of Lanczos to get low mode basis
+  BlockImplicitlyRestartedLanczos<LatticeFermion> IRL1(Op1,Op1test,Nstop1,Nk1,Nm1,resid1,betastp1,MaxIt,MinRes1);
+  int Nconv;
+
+  char tag[1024];
+  if (!FieldVectorIO::read_argonne(evec,(char *)"checkpoint") || !read_evals(UGrid,(char *)"checkpoint/eigen-values.txt",eval1)) {
+
+    if (simple_krylov_basis) {
+      quick_krylov_basis(evec,src,Op1,Nstop1);
+    } else {
+      IRL1.calc(eval1,evec,src,Nconv,false,1);
+    }
+    evec.resize(Nstop1); // and throw away superfluous
+    eval1.resize(Nstop1);
+    if (checkpoint_basis)
+      FieldVectorIO::write_argonne(evec,(char *)"checkpoint");
+    if (UGrid->IsBoss() && checkpoint_basis)
+      write_evals((char *)"checkpoint/eigen-values.txt",eval1);
+
+    Ddwf.Report();
+
+    if (exit_after_basis_calculation) {
+      Grid_finalize();
+      return 0;
+    }
+  }
+
+  // now test eigenvectors
+  if (!simple_krylov_basis) {
+    for (int i=0;i<Nstop1;i++){
+      auto B = evec[i];
+      auto tmp = B;
+      auto v = B;
+      
+      {
+	HermOp.HermOp(B,v);
+	
+	RealD vnum = real(innerProduct(B,v)); // HermOp.
+	RealD vden = norm2(B);
+	RealD vv0 = norm2(v);
+	RealD eval2 = vnum/vden;
+	v -= eval2*B;
+	RealD vv = norm2(v);
+	
+	std::cout << i << " OP eval = " << eval2 << " (" << eval1[i] << ") "
+		  << "res2 = " << vv << " norm2 = " << norm2(B) << std::endl;
+      }
+    }
+  }
+
+  // do second step only if needed
+  if (Nstop1 <= Nstop2) {
+    
+    // Now setup blocking
+    assert(evec.size() == Nstop1);
+    BlockedGrid<LatticeFermion> bgrid(FrbGrid, block_size);
+    BlockProjector<LatticeFermion> pr(evec,bgrid);
+    pr.createOrthonormalBasis(basis_norm_threshold);
+    pr.createOrthonormalBasis(basis_norm_threshold); // another round due to precision issues created by local coherence
+
+    constexpr int common_basis_sizes[] = { 60, 250, 400 };
+    constexpr int n_common_basis_sizes = sizeof(common_basis_sizes) / sizeof(common_basis_sizes[0]);
+    switch (Nstop1) {
+#define BASIS(n) case common_basis_sizes[n]:\
+      CoarseGridLanczos<LatticeFermion,common_basis_sizes[n]>\
+	(pr,alpha2,beta,Npoly2,Nstop2,Nk2,Nm2,resid2,betastp2,MaxIt,MinRes2,HermOp,eval1, \
+	 cg_test_enabled,cg_test_maxiter,nsingle,SkipTest2, \
+	 MaxApply2,smoothed_eval_enabled,smoothed_eval_inner,smoothed_eval_outer, \
+	 smoothed_eval_begin,smoothed_eval_end,smoothed_eval_inner_resid); break;
+      BASIS(0);
+      BASIS(1);
+      BASIS(2);
+    default:
+      std::cout << GridLogMessage << "Basis size " << Nstop1 << " must be added at compile-time" << std::endl;
+      std::cout << GridLogMessage << "Currently available sizes: " << std::endl;
+      for (int i=0;i<n_common_basis_sizes;i++) {
+	std::cout << GridLogMessage << "  " << common_basis_sizes[i] << std::endl;
+      }
+    }
+
+  }
+    
+  Grid_finalize();
+}
+

From 9fe6ac71eab8e0876f1fac6e8d753663d72bcfc7 Mon Sep 17 00:00:00 2001
From: paboyle <paboyle@ph.ed.ac.uk>
Date: Wed, 11 Oct 2017 10:12:07 +0100
Subject: [PATCH 114/115] Starting reorg of Blocked lanczos

---
 configure.ac                                                  | 1 +
 .../BlockImplicitlyRestartedLanczos.h                         | 1 -
 tests/Makefile.am                                             | 2 +-
 .../lanczos}/FieldVectorIO.h                                  | 0
 tests/lanczos/Makefile.am                                     | 1 +
 tests/{solver => lanczos}/Params.h                            | 0
 tests/{solver => lanczos}/Test_dwf_compressed_lanczos.cc      | 4 ++--
 tests/{solver => lanczos}/Test_dwf_lanczos.cc                 | 0
 tests/{debug => lanczos}/Test_synthetic_lanczos.cc            | 0
 tests/{solver => lanczos}/Test_wilson_lanczos.cc              | 0
 10 files changed, 5 insertions(+), 4 deletions(-)
 rename {lib/algorithms/iterative/BlockImplicitlyRestartedLanczos => tests/lanczos}/FieldVectorIO.h (100%)
 create mode 100644 tests/lanczos/Makefile.am
 rename tests/{solver => lanczos}/Params.h (100%)
 rename tests/{solver => lanczos}/Test_dwf_compressed_lanczos.cc (99%)
 rename tests/{solver => lanczos}/Test_dwf_lanczos.cc (100%)
 rename tests/{debug => lanczos}/Test_synthetic_lanczos.cc (100%)
 rename tests/{solver => lanczos}/Test_wilson_lanczos.cc (100%)

diff --git a/configure.ac b/configure.ac
index b11d6b42..496f7fd7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -550,6 +550,7 @@ AC_CONFIG_FILES(tests/forces/Makefile)
 AC_CONFIG_FILES(tests/hadrons/Makefile)
 AC_CONFIG_FILES(tests/hmc/Makefile)
 AC_CONFIG_FILES(tests/solver/Makefile)
+AC_CONFIG_FILES(tests/lanczos/Makefile)
 AC_CONFIG_FILES(tests/smearing/Makefile)
 AC_CONFIG_FILES(tests/qdpxx/Makefile)
 AC_CONFIG_FILES(tests/testu01/Makefile)
diff --git a/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h
index 82a00efa..55a85552 100644
--- a/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h
+++ b/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h
@@ -39,7 +39,6 @@ Author: Christoph Lehner <clehner@bnl.gov>
 #include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockedGrid.h>
 #include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldBasisVector.h>
 #include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockProjector.h>
-#include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldVectorIO.h>
 
 namespace Grid { 
 
diff --git a/tests/Makefile.am b/tests/Makefile.am
index a8935268..7928a7fe 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,4 +1,4 @@
-SUBDIRS = . core forces hmc solver debug smearing IO
+SUBDIRS = . core forces hmc solver debug smearing IO lanczos
 
 if BUILD_CHROMA_REGRESSION
   SUBDIRS+= qdpxx
diff --git a/lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldVectorIO.h b/tests/lanczos/FieldVectorIO.h
similarity index 100%
rename from lib/algorithms/iterative/BlockImplicitlyRestartedLanczos/FieldVectorIO.h
rename to tests/lanczos/FieldVectorIO.h
diff --git a/tests/lanczos/Makefile.am b/tests/lanczos/Makefile.am
new file mode 100644
index 00000000..60b82dd7
--- /dev/null
+++ b/tests/lanczos/Makefile.am
@@ -0,0 +1 @@
+include Make.inc
diff --git a/tests/solver/Params.h b/tests/lanczos/Params.h
similarity index 100%
rename from tests/solver/Params.h
rename to tests/lanczos/Params.h
diff --git a/tests/solver/Test_dwf_compressed_lanczos.cc b/tests/lanczos/Test_dwf_compressed_lanczos.cc
similarity index 99%
rename from tests/solver/Test_dwf_compressed_lanczos.cc
rename to tests/lanczos/Test_dwf_compressed_lanczos.cc
index b42a2d55..7fe37387 100644
--- a/tests/solver/Test_dwf_compressed_lanczos.cc
+++ b/tests/lanczos/Test_dwf_compressed_lanczos.cc
@@ -21,9 +21,9 @@
     (ortho krylov low poly); and then fix up lowest say 200 eigenvalues by 1 run with high-degree poly (600 could be enough)
 */
 #include <Grid/Grid.h>
-#include "Params.h"
-
 #include <Grid/algorithms/iterative/BlockImplicitlyRestartedLanczos/BlockImplicitlyRestartedLanczos.h>
+#include "FieldVectorIO.h"
+#include "Params.h"
 
 using namespace std;
 using namespace Grid;
diff --git a/tests/solver/Test_dwf_lanczos.cc b/tests/lanczos/Test_dwf_lanczos.cc
similarity index 100%
rename from tests/solver/Test_dwf_lanczos.cc
rename to tests/lanczos/Test_dwf_lanczos.cc
diff --git a/tests/debug/Test_synthetic_lanczos.cc b/tests/lanczos/Test_synthetic_lanczos.cc
similarity index 100%
rename from tests/debug/Test_synthetic_lanczos.cc
rename to tests/lanczos/Test_synthetic_lanczos.cc
diff --git a/tests/solver/Test_wilson_lanczos.cc b/tests/lanczos/Test_wilson_lanczos.cc
similarity index 100%
rename from tests/solver/Test_wilson_lanczos.cc
rename to tests/lanczos/Test_wilson_lanczos.cc

From 27936900e63c51aa6d52d2cb2cfdfa2936adbbd3 Mon Sep 17 00:00:00 2001
From: Guido Cossu <guido.cossu@ed.ac.uk>
Date: Wed, 18 Oct 2017 13:08:09 +0100
Subject: [PATCH 115/115] Putting the FG verbosity in the Integrator level

---
 lib/qcd/hmc/integrators/Integrator_algorithm.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/qcd/hmc/integrators/Integrator_algorithm.h b/lib/qcd/hmc/integrators/Integrator_algorithm.h
index ecc125ef..13a37aeb 100644
--- a/lib/qcd/hmc/integrators/Integrator_algorithm.h
+++ b/lib/qcd/hmc/integrators/Integrator_algorithm.h
@@ -231,7 +231,7 @@ class ForceGradient : public Integrator<FieldImplementation, SmearingPolicy,
     Field Pfg(U._grid);
     Ufg = U;
     Pfg = zero;
-    std::cout << GridLogMessage << "FG update " << fg_dt << " " << ep
+    std::cout << GridLogIntegrator << "FG update " << fg_dt << " " << ep
               << std::endl;
     // prepare_fg; no prediction/result cache for now
     // could relax CG stopping conditions for the