From 344631ac1033779329f0aa339e128826019e11b0 Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Fri, 17 Mar 2023 11:56:38 -0700
Subject: [PATCH 01/12] wip

---
 dbldatagen/text_generators.py |  14 +-
 tests/test_text_templates.py  | 276 ++++++++++++++++++++++++++++++++++
 2 files changed, 288 insertions(+), 2 deletions(-)
 create mode 100644 tests/test_text_templates.py

diff --git a/dbldatagen/text_generators.py b/dbldatagen/text_generators.py
index 96c03a4e..4def1f70 100644
--- a/dbldatagen/text_generators.py
+++ b/dbldatagen/text_generators.py
@@ -11,6 +11,7 @@
 
 import numpy as np
 import pandas as pd
+import logging
 
 #: list of hex digits for template generation
 _HEX_LOWER = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
@@ -243,9 +244,18 @@ def __init__(self, template, escapeSpecialChars=False, extendedWordList=None):
         self._np_letters_all = np.array(_LETTERS_ALL)
         self._lenWords = len(self._wordList)
 
-        # get the template metadata
+        # get the template metadata - this will be list of metadata entries for each template
+        # for each template, metadata will be tuple of number of placeholders followed by list of random bounds
+        # to be computed when replacing non static placeholder
         template_info =  [self._prepareTemplateStrings(template, escapeSpecialMeaning=escapeSpecialChars)
                                     for template in self._templates]
+
+        logger = logging.getLogger(__name__)
+
+        #if logger.isEnabledFor(logging.DEBUG):
+        for ix, ti in template_info:
+            logger.info(f"templates - {ix} {ti}")
+
         self._max_placeholders = max([ x[0] for x in template_info])
         self._max_rnds_needed = max([ len(x[1]) for x in template_info])
         self._placeholders_needed = [ x[0] for x in template_info]
@@ -637,7 +647,7 @@ def pandasGenerateText(self, v):
             for m in masked_matrices:
                 np.ma.harden_mask(m)
 
-            # expand values into placeholders
+            # expand values into placeholders without affect masked values
             #self._applyTemplateStringsForTemplate(v.to_numpy(dtype=np.object_), #masked_base_values,
             self._applyTemplateStringsForTemplate(v,
                                                     #masked_base_values,
diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
new file mode 100644
index 00000000..0968724b
--- /dev/null
+++ b/tests/test_text_templates.py
@@ -0,0 +1,276 @@
+import re
+import pytest
+import pandas as pd
+import numpy as np
+
+import pyspark.sql.functions as F
+from pyspark.sql.types import BooleanType, DateType
+from pyspark.sql.types import StructType, StructField, IntegerType, StringType, TimestampType
+
+import dbldatagen as dg
+from dbldatagen import TemplateGenerator, TextGenerator
+
+# add the following if using pandas udfs
+#    .config("spark.sql.execution.arrow.maxRecordsPerBatch", "1000") \
+
+
+spark = dg.SparkSingleton.getLocalInstance("unit tests")
+
+spark.conf.set("spark.sql.execution.arrow.maxRecordsPerBatch", "20000")
+spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
+
+
+# Test manipulation and generation of test data for a large schema
+class TestTextTemplates:
+    testDataSpec = None
+    row_count = 100000
+    partitions_requested = 4
+
+    @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
+                             [ (r'\\w \\w|\\w A. \\w', False, False),
+                               (r'\w \w|\w A. \w', False, False),
+                               (r'\w A. \w', False, False),
+                               (r'\\w \\w|\\w a. \\w', False, False),
+                                (r'\\w \\w|\\w k. \\w', False, False),
+                                (r'\\w \\w|\\w K. \\w', False, False),
+                                (r'\\w \\w|\\w x. \\w', False, False),
+                                (r'\\w \\w|\\w X. \\w', False, False),
+                               (r'\\w \\w|\\w A. \\w', False, True),
+                               (r'\\w \\w|\\w a. \\w', False, True),
+                               (r'\\w \\w|\\w k. \\w', False, True),
+                               (r'\\w \\w|\\w K. \\w', False, True),
+                               (r'\\w \\w|\\w x. \\w', False, True),
+                               (r'\\w \\w|\\w X. \\w', False, True),
+                               (r'\\w \\w|\\w A. \\w', True, True),
+                               (r'\w \w|\w \A. \w', True, True),
+                               (r'\\w \\w|\\w a. \\w', True, True),
+                               (r'\w \w|\w \a. \w', True, True),
+                               (r'\\w \\w|\\w k. \\w', True, True),
+                               (r'\\w \\w|\\w K. \\w', True, True),
+                               (r'\\w \\w|\\w x. \\w', True, True),
+                               (r'\\w \\w|\\w X. \\w', True, True),
+                               (r'\\w a. \\w|\\w \\w', False, False),
+                               (r'\\w k. \\w|\\w \\w', False, False),
+                               (r'\\w a. \\w', False, True),
+                               (r'\\w k. \\w', False, True),
+                               (r'\w \a. \w', True, True),
+                               (r'\w \k. \w', True, True),
+
+                               ])
+
+    def test_rnd_compute(self, template_provided, escapeSpecial, useTemplateObject):
+        template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
+        print(f"template [{template_provided}]")
+
+        arr = np.arange(100)
+
+        template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
+
+        assert len(template_choices) == len(template_rnds)
+        assert len(template_choices) == len(template_rnd_bounds)
+
+        for ix in range(len(template_choices)):
+            bounds = template_rnd_bounds[ix]
+            rnds = template_rnds[ix]
+
+            assert len(bounds) == len(rnds)
+
+            for iy in range(len(bounds)):
+                assert bounds[iy] == -1 or (rnds[iy] < bounds[iy])
+
+    @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
+                             [ (r'\\w \\w|\\w A. \\w|\w n n \w', False, False),
+                               (r'\w \w|\w A. \w', False, False),
+                               (r'\w A. \w', False, False),
+                               (r'\\w \\w|\\w a. \\w', False, False),
+                                (r'\\w \\w|\\w k. \\w', False, False),
+                                (r'\\w \\w|\\w K. \\w', False, False),
+                                (r'\\w \\w|\\w x. \\w', False, False),
+                                (r'\\w \\w|\\w X. \\w', False, False),
+                               (r'\\w \\w|\\w A. \\w', False, True),
+                               (r'\\w \\w|\\w a. \\w', False, True),
+                               (r'\\w \\w|\\w k. \\w', False, True),
+                               (r'\\w \\w|\\w K. \\w', False, True),
+                               (r'\\w \\w|\\w x. \\w', False, True),
+                               (r'\\w \\w|\\w X. \\w', False, True),
+                               (r'\\w \\w|\\w A. \\w', True, True),
+                               (r'\w \w|\w \A. \w', True, True),
+                               (r'\\w \\w|\\w a. \\w', True, True),
+                               (r'\w \w|\w \a. \w', True, True),
+                               (r'\\w \\w|\\w k. \\w', True, True),
+                               (r'\\w \\w|\\w K. \\w', True, True),
+                               (r'\\w \\w|\\w x. \\w', True, True),
+                               (r'\\w \\w|\\w X. \\w', True, True),
+                               (r'\\w a. \\w|\\w \\w', False, False),
+                               (r'\\w k. \\w|\\w \\w', False, False),
+                               (r'\\w a. \\w', False, True),
+                               (r'\\w k. \\w', False, True),
+                               (r'\w \a. \w', True, True),
+                               (r'\w \k. \w', True, True),
+
+                               ])
+
+    def test_masking(self, template_provided, escapeSpecial, useTemplateObject):
+        template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
+        print(f"template [{template_provided}]")
+
+        arr = np.arange(100)
+
+        template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
+
+        assert len(template_choices) == len(template_rnds)
+        assert len(template_choices) == len(template_rnd_bounds)
+
+        arr2 = np.full(shape=(arr.shape[0], 2), fill_value="testing 1 2 3 4 ", dtype=np.object_)
+        print(template_choices)
+
+        template_choices_t = template_choices.T
+
+        masked_placeholders = np.ma.MaskedArray(arr2, mask=False)
+        for x in range(len(template1._templates)-1):
+            masked_placeholders[template_choices_t != x] = np.ma.masked
+            np.ma.harden_mask(masked_placeholders)
+
+            masked_placeholders[:, 0] = template1.templates[x]
+            masked_placeholders[:, 1] = template1.templates[x]
+
+            np.ma.soften_mask(masked_placeholders)
+            masked_placeholders.mask = False
+
+
+        print(arr2)
+
+
+
+
+    @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
+                             [ (r'\\w \\w|\\w A. \\w', False, False),
+                               (r'\w \w|\w A. \w', False, False),
+                               (r'\w A. \w', False, False),
+                               (r'\\w \\w|\\w a. \\w', False, False),
+                                (r'\\w \\w|\\w k. \\w', False, False),
+                                (r'\\w \\w|\\w K. \\w', False, False),
+                                (r'\\w \\w|\\w x. \\w', False, False),
+                                (r'\\w \\w|\\w X. \\w', False, False),
+                               (r'\\w \\w|\\w A. \\w', False, True),
+                               (r'\\w \\w|\\w a. \\w', False, True),
+                               (r'\\w \\w|\\w k. \\w', False, True),
+                               (r'\\w \\w|\\w K. \\w', False, True),
+                               (r'\\w \\w|\\w x. \\w', False, True),
+                               (r'\\w \\w|\\w X. \\w', False, True),
+                               (r'\\w \\w|\\w A. \\w', True, True),
+                               (r'\w \w|\w \A. \w', True, True),
+                               (r'\\w \\w|\\w a. \\w', True, True),
+                               (r'\w \w|\w \a. \w', True, True),
+                               (r'\\w \\w|\\w k. \\w', True, True),
+                               (r'\\w \\w|\\w K. \\w', True, True),
+                               (r'\\w \\w|\\w x. \\w', True, True),
+                               (r'\\w \\w|\\w X. \\w', True, True),
+                               (r'\\w a. \\w|\\w \\w', False, False),
+                               (r'\\w k. \\w|\\w \\w', False, False),
+                               (r'\\w a. \\w', False, True),
+                               (r'\\w k. \\w', False, True),
+                               (r'\w \a. \w', True, True),
+                               (r'\w \k. \w', True, True),
+
+                               ])
+
+    def test_text_templates1(self, template_provided, escapeSpecial, useTemplateObject):
+        template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
+        print(f"template [{template_provided}]")
+
+        print("max_placeholders", template1._max_placeholders )
+        print("max_rnds", template1._max_rnds_needed)
+        print("placeholders", template1._placeholders_needed )
+        print("bounds", template1._template_rnd_bounds)
+
+        print("templates", template1.templates)
+
+        arr = np.arange(100)
+
+        template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
+
+        print("choices", template_choices)
+        print("rnd bounds", template_rnd_bounds)
+        print("template_rnds", template_rnds)
+
+        assert len(template_choices) == len(template_rnds)
+        assert len(template_choices) == len(template_rnd_bounds)
+
+        for ix in range(len(template_choices)):
+            bounds = template_rnd_bounds[ix]
+            rnds = template_rnds[ix]
+
+            assert len(bounds) == len(rnds)
+
+            for iy in range(len(bounds)):
+                assert bounds[iy] == -1 or (rnds[iy] < bounds[iy])
+
+
+        results = template1.pandasGenerateText(arr)
+        print(results)
+
+
+
+
+
+
+    @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
+                             [ (r'\\w \\w|\\w A. \\w', False, False),
+                               (r'\w \w|\w A. \w', False, False),
+                               (r'\w A. \w', False, False),
+                               (r'\\w \\w|\\w a. \\w', False, False),
+                                (r'\\w \\w|\\w k. \\w', False, False),
+                                (r'\\w \\w|\\w K. \\w', False, False),
+                                (r'\\w \\w|\\w x. \\w', False, False),
+                                (r'\\w \\w|\\w X. \\w', False, False),
+                               (r'\\w \\w|\\w A. \\w', False, True),
+                               (r'\\w \\w|\\w a. \\w', False, True),
+                               (r'\\w \\w|\\w k. \\w', False, True),
+                               (r'\\w \\w|\\w K. \\w', False, True),
+                               (r'\\w \\w|\\w x. \\w', False, True),
+                               (r'\\w \\w|\\w X. \\w', False, True),
+                               (r'\\w \\w|\\w A. \\w', True, True),
+                               (r'\w \w|\w \A. \w', True, True),
+                               (r'\\w \\w|\\w a. \\w', True, True),
+                               (r'\w \w|\w \a. \w', True, True),
+                               (r'\\w \\w|\\w k. \\w', True, True),
+                               (r'\\w \\w|\\w K. \\w', True, True),
+                               (r'\\w \\w|\\w x. \\w', True, True),
+                               (r'\\w \\w|\\w X. \\w', True, True),
+                               (r'\\w a. \\w|\\w \\w', False, False),
+                               (r'\\w k. \\w|\\w \\w', False, False),
+                               (r'\\w a. \\w', False, True),
+                               (r'\\w k. \\w', False, True),
+                               (r'\w \a. \w', True, True),
+                               (r'\w \k. \w', True, True),
+                               (r'\w \w|\w \w \w|\w \n \w|\w \w \w \w', True, True),
+                               (r'\w \n \w', True, True),
+
+                               ])
+
+    def test_text_templates2(self, template_provided, escapeSpecial, useTemplateObject):
+        import dbldatagen as dg
+        print(f"template [{template_provided}]")
+
+        data_rows = 10 * 1000
+
+        uniqueCustomers = 10 * 1000
+
+        dataspec = (dg.DataGenerator(spark, rows=data_rows, partitions=4)
+                    .withColumn("customer_id", "long", uniqueValues=uniqueCustomers)
+                    )
+
+        if useTemplateObject or escapeSpecial:
+            template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
+            dataspec = dataspec.withColumn("name", percentNulls=0.01, text=template1)
+        else:
+            dataspec = dataspec.withColumn("name", percentNulls=0.01, template=template_provided)
+
+        df1 = dataspec.build()
+        df1.show()
+
+        count = df1.where("name is not null").count()
+        assert count > 0
+
+

From b6e1eb306be69900f7a5933a98da7ab6464a5f2a Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Fri, 17 Mar 2023 14:49:45 -0700
Subject: [PATCH 02/12] merged updates to master

---
 CHANGELOG.md                  |   5 ++
 README.md                     |  38 +++++----
 dbldatagen/_version.py        |   3 +-
 dbldatagen/data_generator.py  |   2 +-
 dbldatagen/spark_singleton.py |   3 +-
 tests/test_logging.py         | 149 ++++++++++++++++++++++++++++++++++
 6 files changed, 183 insertions(+), 17 deletions(-)
 create mode 100644 tests/test_logging.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b5450447..e8866f1f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,11 @@
 ## Change History
 All notable changes to the Databricks Labs Data Generator will be documented in this file.
 
+### Unreleased
+
+#### Changed
+* Fixed use of logger in _version.py and in spark_singleton.py
+
 ### Version 0.3.2
 
 #### Changed
diff --git a/README.md b/README.md
index 2e8ff2ab..a2ec0e10 100644
--- a/README.md
+++ b/README.md
@@ -9,8 +9,10 @@
 <!-- Dont remove: end exclude package -->
 
 [![build](https://github.com/databrickslabs/dbldatagen/workflows/build/badge.svg?branch=master)](https://github.com/databrickslabs/dbldatagen/actions?query=workflow%3Abuild+branch%3Amaster)
+[![PyPi package](https://img.shields.io/pypi/v/dbldatagen?color=green)](https://pypi.org/project/dbldatagen/)
 [![codecov](https://codecov.io/gh/databrickslabs/dbldatagen/branch/master/graph/badge.svg)](https://codecov.io/gh/databrickslabs/dbldatagen)
-[![PyPi downloads](https://img.shields.io/pypi/dm/dbldatagen?label=PyPi%20Downloads)](https://pypi.org/project/dbldatagen/)
+[![PyPi downloads](https://img.shields.io/pypi/dm/dbldatagen?label=PyPi%20Downloads)](https://pypistats.org/packages/dbldatagen)
+
 <!-- 
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/databrickslabs/dbldatagen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/databrickslabs/dbldatagen/context:python)
 [![downloads](https://img.shields.io/github/downloads/databrickslabs/dbldatagen/total.svg)](https://hanadigital.github.io/grev/?user=databrickslabs&repo=dbldatagen)
@@ -89,6 +91,14 @@ release notes for library compatibility
 
 - https://docs.databricks.com/release-notes/runtime/releases.html
 
+When using the Databricks Labs Data Generator on Unity Catalog enabled environments, the Data Generator requires
+the use of `Single User` or `No Isolation Shared` access modes as some needed features are not available in `Shared` 
+mode (for example, use of 3rd party libraries). Depending on settings, `Custom` access mode may be supported.
+
+See the following documentation for more information:
+
+- https://docs.databricks.com/data-governance/unity-catalog/compute.html
+
 ## Using the Data Generator
 To use the data generator, install the library using the `%pip install` method or install the Python wheel directly 
 in your environment.
@@ -104,19 +114,19 @@ column_count = 10
 data_rows = 1000 * 1000
 df_spec = (dg.DataGenerator(spark, name="test_data_set1", rows=data_rows,
                                                   partitions=4)
-                            .withIdOutput()
-                            .withColumn("r", FloatType(), 
-                                             expr="floor(rand() * 350) * (86400 + 3600)",
-                                             numColumns=column_count)
-                            .withColumn("code1", IntegerType(), minValue=100, maxValue=200)
-                            .withColumn("code2", IntegerType(), minValue=0, maxValue=10)
-                            .withColumn("code3", StringType(), values=['a', 'b', 'c'])
-                            .withColumn("code4", StringType(), values=['a', 'b', 'c'], 
-                                           random=True)
-                            .withColumn("code5", StringType(), values=['a', 'b', 'c'], 
-                                           random=True, weights=[9, 1, 1])
-
-                            )
+           .withIdOutput()
+           .withColumn("r", FloatType(), 
+                            expr="floor(rand() * 350) * (86400 + 3600)",
+                            numColumns=column_count)
+           .withColumn("code1", IntegerType(), minValue=100, maxValue=200)
+           .withColumn("code2", IntegerType(), minValue=0, maxValue=10)
+           .withColumn("code3", StringType(), values=['a', 'b', 'c'])
+           .withColumn("code4", StringType(), values=['a', 'b', 'c'], 
+                          random=True)
+           .withColumn("code5", StringType(), values=['a', 'b', 'c'], 
+                          random=True, weights=[9, 1, 1])
+ 
+           )
                             
 df = df_spec.build()
 num_rows=df.count()                          
diff --git a/dbldatagen/_version.py b/dbldatagen/_version.py
index 0fdb74ce..0f96dc86 100644
--- a/dbldatagen/_version.py
+++ b/dbldatagen/_version.py
@@ -44,7 +44,8 @@ def _get_spark_version(sparkVersion):
         spark_version_info = VersionInfo(int(major), int(minor), int(patch), release, build="0")
     except (RuntimeError, AttributeError):
         spark_version_info = VersionInfo(major=3, minor=0, patch=1, release="unknown", build="0")
-        logging.warning("Could not parse spark version - using assumed Spark Version : %s", spark_version_info)
+        logger = logging.getLogger(__name__)
+        logger.warning("Could not parse spark version - using assumed Spark Version : %s", spark_version_info)
 
     return spark_version_info
 
diff --git a/dbldatagen/data_generator.py b/dbldatagen/data_generator.py
index c8c5c964..f84d8fb0 100644
--- a/dbldatagen/data_generator.py
+++ b/dbldatagen/data_generator.py
@@ -58,7 +58,7 @@ class DataGenerator:
 
     # restrict spurious messages from java gateway
     logging.getLogger("py4j").setLevel(logging.WARNING)
-    logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.NOTSET)
+    #logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.NOTSET)
 
     def __init__(self, sparkSession=None, name=None, randomSeedMethod=None,
                  rows=1000000, startingId=0, randomSeed=None, partitions=None, verbose=False,
diff --git a/dbldatagen/spark_singleton.py b/dbldatagen/spark_singleton.py
index b6bb216c..5e61c356 100644
--- a/dbldatagen/spark_singleton.py
+++ b/dbldatagen/spark_singleton.py
@@ -42,7 +42,8 @@ def getLocalInstance(cls, appName="new Spark session", useAllCores=True):
         else:
             spark_core_count = cpu_count - 1
 
-        logging.info("Spark core count: %d", spark_core_count)
+        logger = logging.getLogger(__name__)
+        logger.info("Spark core count: %d", spark_core_count)
 
         sparkSession = SparkSession.builder \
             .master(f"local[{spark_core_count}]") \
diff --git a/tests/test_logging.py b/tests/test_logging.py
new file mode 100644
index 00000000..6daab42e
--- /dev/null
+++ b/tests/test_logging.py
@@ -0,0 +1,149 @@
+import logging
+import pytest
+
+from pyspark.sql import functions as F
+from pyspark.sql.types import StructType, StructField, IntegerType, StringType, FloatType
+
+#import  dbldatagen as dg
+
+
+@pytest.fixture(scope="class")
+def setupSpark():
+    import dbldatagen as dg
+    sparkSession = dg.SparkSingleton.getLocalInstance("unit tests")
+    return sparkSession
+
+
+@pytest.fixture(scope="class")
+def setupLogging():
+    #FORMAT = '%(asctime)-15s %(message)s'
+    #logging.basicConfig(format=FORMAT)
+    pass
+
+
+class TestLoggingOperation:
+    testDataSpec = None
+    dfTestData = None
+    SMALL_ROW_COUNT = 100000
+    TINY_ROW_COUNT = 1000
+    column_count = 10
+    row_count = SMALL_ROW_COUNT
+
+    def setup_log_capture(self, caplog_object):
+        """ set up log capture fixture
+
+        Sets up log capture fixture to only capture messages after setup and only
+        capture warnings and errors
+
+        """
+        caplog_object.set_level(logging.INFO)
+
+        # clear messages from setup
+        caplog_object.clear()
+
+    def get_log_capture_warnings_and_errors(self, caplog_object, textFlag):
+        """
+        gets count of errors containing specified text
+
+        :param caplog_object: log capture object from fixture
+        :param textFlag: text to search for to include error or warning in count
+        :return: count of errors containg text specified in `textFlag`
+        """
+        flagged_text_warnings_and_errors = 0
+        for r in caplog_object.records:
+            if (r.levelname == "WARNING" or r.levelname == "ERROR") and textFlag in r.message:
+                flagged_text_warnings_and_errors += 1
+
+        return flagged_text_warnings_and_errors
+
+    def get_log_capture_info(self, caplog_object, textFlag):
+        """
+        gets count of errors containing specified text
+
+        :param caplog_object: log capture object from fixture
+        :param textFlag: text to search for to include error or warning in count
+        :return: count of errors containg text specified in `textFlag`
+        """
+        flagged_text_info = 0
+        for r in caplog_object.records:
+            if (r.levelname == "INFO") and textFlag in r.message:
+                flagged_text_info += 1
+
+        return flagged_text_info
+
+
+    def test_logging_operation(self, caplog):
+        # caplog fixture captures log content
+        self.setup_log_capture(caplog)
+
+        date_format = "%Y-%m-%d %H:%M:%S"
+        log_format = "[%(name)s]%(asctime)s %(levelname)-8s [%(module)s][%(funcName)s] TESTING1 %(message)s"
+        formatter = logging.Formatter(log_format, date_format)
+        handler = logging.StreamHandler()
+        handler.setFormatter(formatter)
+        logger = logging.getLogger(__name__)
+        logger.setLevel(level=logging.INFO)
+        logger.addHandler(handler)
+
+        logger.warning("Info message 1")
+
+        # Prints: 2023-03-08 14:18:59 INFO      Info message
+
+        from dbldatagen import DataGenerator
+
+        logger.info("Info message 2")
+
+        for h in logger.handlers:
+            h.flush()
+
+        message1_count = self.get_log_capture_warnings_and_errors(caplog, "Info message 1")
+        assert message1_count == 1, "Should only have 1 message 1"
+
+        message2_count = self.get_log_capture_info(caplog, "Info message 2")
+        assert message2_count == 1, "Should only have 1 message 2"
+
+
+    def test_logging_operation2(self, setupSpark, caplog):
+        self.setup_log_capture(caplog)
+
+        spark=setupSpark
+
+        date_format = "%Y-%m-%d %H:%M:%S"
+        log_format = "%(asctime)s %(levelname)-8s TESTING2  %(message)s"
+        formatter1 = logging.Formatter(log_format, date_format)
+        handler1 = logging.StreamHandler()
+        handler1.setFormatter(formatter1)
+        logger2 = logging.getLogger("test1")
+        logger2.setLevel(level=logging.INFO)
+        logger2.addHandler(handler1)
+
+        logger2.warning("Info message 1")
+        # Prints: 2023-03-08 14:18:59 INFO      Info message
+
+        from dbldatagen import DataGenerator
+
+        spec = (DataGenerator(sparkSession=spark, name="test_data_set1", rows=10000, seedMethod='hash_fieldname')
+                .withIdOutput()
+                .withColumn("r", "float", expr="floor(rand() * 350) * (86400 + 3600)",
+                            numColumns=10)
+                .withColumn("code1", "int", min=100, max=200)
+                .withColumn("code2", "int", min=0, max=10)
+                .withColumn("code3", "string", values=['a', 'b', 'c'])
+                .withColumn("code4", "string", values=['a', 'b', 'c'], random=True)
+                .withColumn("code5", "string", values=['a', 'b', 'c'], random=True, weights=[9, 1, 1])
+                )
+
+        df = spec.build()
+
+
+        # Prints: INFO: Version : VersionInfo(major='0', minor='3', patch='1', release='', build='')
+        logger2.info("Info message 2")
+
+        for h in logger2.handlers:
+            h.flush()
+
+        message1_count = self.get_log_capture_warnings_and_errors(caplog, "Info message 1")
+        assert message1_count == 1, "Should only have 1 message 1"
+
+        message2_count = self.get_log_capture_info(caplog, "Info message 2")
+        assert message2_count == 1, "Should only have 1 message 2"

From 66f21b2a301b72951138d062366668c8bcc96943 Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Sun, 19 Mar 2023 19:33:09 -0700
Subject: [PATCH 03/12] wip

---
 CHANGELOG.md                  |   1 +
 dbldatagen/text_generators.py | 176 ++++++++++++++--------------------
 tests/test_text_templates.py  |  13 ++-
 3 files changed, 81 insertions(+), 109 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e8866f1f..e89ad16d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@ All notable changes to the Databricks Labs Data Generator will be documented in
 
 #### Changed
 * Fixed use of logger in _version.py and in spark_singleton.py
+* Fixed template issues 
 
 ### Version 0.3.2
 
diff --git a/dbldatagen/text_generators.py b/dbldatagen/text_generators.py
index 4def1f70..404f7ea8 100644
--- a/dbldatagen/text_generators.py
+++ b/dbldatagen/text_generators.py
@@ -244,6 +244,24 @@ def __init__(self, template, escapeSpecialChars=False, extendedWordList=None):
         self._np_letters_all = np.array(_LETTERS_ALL)
         self._lenWords = len(self._wordList)
 
+        self._templateMappings = {
+            'a': (26, self._np_letters_lower),
+            'A': (26, self._np_letters_upper),
+            'x': (16, self._np_hex_lower),
+            'X': (16, self._np_hex_upper),
+            'd': (10, self._np_digits_zero),
+            'D': (9, self._np_digits_non_zero),
+            'k': (36, self._np_alnum_lower),
+            'K': (36, self._np_alnum_upper)
+        }
+
+        self._templateEscapedMappings = {
+            'n': (256, None),
+            'N': (65536, None),
+            'w': (self._lenWords, self._wordList),
+            'W': (self._lenWords, self._upperWordList)
+        }
+
         # get the template metadata - this will be list of metadata entries for each template
         # for each template, metadata will be tuple of number of placeholders followed by list of random bounds
         # to be computed when replacing non static placeholder
@@ -329,57 +347,17 @@ def _prepareTemplateStrings(self, genTemplate, escapeSpecialMeaning=False):
                 # retval.append(str(baseValue[val_index]))
                 num_placeholders += 1
                 use_value = False
-            elif char == 'x' and (not escape) ^ escapeSpecialMeaning:
-                retval.append(16)
-                num_placeholders += 1
-                # used for retval.append(_HEX_LOWER[self._getRandomInt(0, 15, rndGenerator)])
-            elif char == 'X' and (not escape) ^ escapeSpecialMeaning:
-                retval.append(16)
-                num_placeholders += 1
-                # retval.append(_HEX_UPPER[self._getRandomInt(0, 15, rndGenerator)])
-            elif char == 'd' and (not escape) ^ escapeSpecialMeaning:
-                retval.append(10)
-                num_placeholders += 1
-                # retval.append(_DIGITS_ZERO[self._getRandomInt(0, 9, rndGenerator)])
-            elif char == 'D' and (not escape) ^ escapeSpecialMeaning:
-                retval.append(9)
-                num_placeholders += 1
-                # retval.append(_DIGITS_NON_ZERO[self._getRandomInt(0, 8, rndGenerator)])
-            elif char == 'a' and (not escape) ^ escapeSpecialMeaning:
-                retval.append(26)
-                num_placeholders += 1
-                # retval.append(_LETTERS_LOWER[self._getRandomInt(0, 25, rndGenerator)])
-            elif char == 'A' and (not escape) ^ escapeSpecialMeaning:
-                retval.append(26)
-                num_placeholders += 1
-                # retval.append(_LETTERS_UPPER[self._getRandomInt(0, 25, rndGenerator)])
-            elif char == 'k' and (not escape) ^ escapeSpecialMeaning:
-                retval.append(26)
-                num_placeholders += 1
-                # retval.append(_ALNUM_LOWER[self._getRandomInt(0, 35, rndGenerator)])
-            elif char == 'K' and (not escape) ^ escapeSpecialMeaning:
-                retval.append(36)
-                num_placeholders += 1
-                # retval.append(_ALNUM_UPPER[self._getRandomInt(0, 35, rndGenerator)])
-            elif char == 'n' and escape:
-                retval.append(256)
-                num_placeholders += 1
-                # retval.append(str(self._getRandomInt(0, 255, rndGenerator)))
-                escape = False
-            elif char == 'N' and escape:
-                retval.append(65536)
+            elif (char in self._templateMappings.keys()) and (not escape) ^ escapeSpecialMeaning:
+                # handle case for ['a','A','k', 'K', 'x', 'X']
+                bound, mappingArr = self._templateMappings[char]
+                retval.append(bound)
                 num_placeholders += 1
-                # retval.append(str(self._getRandomInt(0, 65535, rndGenerator)))
                 escape = False
-            elif char == 'W' and escape:
-                retval.append(self._lenWords)
+            elif (char in self._templateEscapedMappings.keys()) and escape:
+                # handle case for ['n', 'N', 'w', 'W']
+                bound, mappingArr = self._templateEscapedMappings[char]
+                retval.append(bound)
                 num_placeholders += 1
-                # retval.append(self._upperWordList[self._getRandomWordOffset(self._lenWords, rndGenerator=rndGenerator)])
-                escape = False
-            elif char == 'w' and escape:
-                retval.append(self._lenWords)
-                num_placeholders += 1
-                # retval.append(self._wordList[self._getRandomWordOffset(self._lenWords, rndGenerator=rndGenerator)])
                 escape = False
             elif char == 'v' and escape:
                 escape = False
@@ -420,12 +398,17 @@ def _applyTemplateStringsForTemplate(self, baseValue, genTemplate, placeholders,
         `_escapeSpecialMeaning` parameter allows for backwards compatibility with old style syntax while allowing
         for preferred new style template syntax. Specify as True to force escapes for special meanings,.
 
+        Note that both placeholders and rnds are masked arrays
+
         """
         assert baseValue.shape[0] == placeholders.shape[0]
         assert baseValue.shape[0] == rnds.shape[0]
 
         _cached_values = {}
 
+        regularKeys = self._templateMappings.keys()
+        escapedKeys = self._templateEscapedMappings.keys()
+
         def _get_values_as_np_array():
             """Get baseValue which is pd.Series or Dataframe as a numpy array and cache it"""
             if "np_values" not in _cached_values:
@@ -454,6 +437,15 @@ def _get_values_subelement(elem):
         num_placeholders = 0
         rnd_offset = 0
 
+        selected = None
+        selected_col0 = None
+
+        assert isinstance(placeholders, np.ma.MaskedArray), "expecting MaskArray"
+
+        if isinstance(placeholders, np.ma.MaskedArray):
+            selected = ~placeholders.mask
+            selected_col0 = selected[:, 1]
+
         # in the following code, the construct `(not escape) ^ self._escapeSpecialMeaning` means apply
         # special meaning if either escape is not true or the option `self._escapeSpecialMeaning` is true.
         # This corresponds to the logical xor operation
@@ -469,72 +461,46 @@ def _get_values_subelement(elem):
                 #placeholders[:, num_placeholders] = pd_base_values.apply(lambda x: str(x[val_index]))
                 num_placeholders += 1
                 use_value = False
-            elif char == 'x' and (not escape) ^ escapeSpecialMeaning:
+            elif char in regularKeys and (not escape) ^ escapeSpecialMeaning:
                 # note vectorized lookup - `rnds[:, rnd_offset]` will get vertical column of
                 # random numbers from `rnds` 2d array
-                placeholders[:, num_placeholders] = self._np_hex_lower[rnds[:, rnd_offset]]
+                bound, valueMappings = self._templateMappings[char]
+
+                print(f"mappings - {bound},{char}", valueMappings)
+                print(f"selected_col0 - {selected_col0}")
+
+                if valueMappings is not None:
+                    if selected_col0 is not None:
+                        placeholders[selected_col0, num_placeholders] = valueMappings[rnds[selected_col0, rnd_offset]]
+                    else:
+                        placeholders[:, num_placeholders] = valueMappings[rnds[:, rnd_offset]]
+                else:
+                    if selected_col0 is not None:
+                        placeholders[selected_col0, num_placeholders] = rnds[selected_col0, rnd_offset]
+                    else:
+                        placeholders[:, num_placeholders] = rnds[:, rnd_offset]
+
                 num_placeholders += 1
                 rnd_offset = rnd_offset + 1
+                escape = False
                 # used for retval.append(_HEX_LOWER[self._getRandomInt(0, 15, rndGenerator)])
-            elif char == 'X' and (not escape) ^ escapeSpecialMeaning:
-                placeholders[:, num_placeholders] = self._np_hex_upper[rnds[:, rnd_offset]]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(_HEX_UPPER[self._getRandomInt(0, 15, rndGenerator)])
-            elif char == 'd' and (not escape) ^ escapeSpecialMeaning:
-                placeholders[:, num_placeholders] = self._np_digits_zero[rnds[:, rnd_offset]]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(_DIGITS_ZERO[self._getRandomInt(0, 9, rndGenerator)])
-            elif char == 'D' and (not escape) ^ escapeSpecialMeaning:
-                placeholders[:, num_placeholders] = self._np_digits_non_zero[rnds[:, rnd_offset]]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(_DIGITS_NON_ZERO[self._getRandomInt(0, 8, rndGenerator)])
-            elif char == 'a' and (not escape) ^ escapeSpecialMeaning:
-                placeholders[:, num_placeholders] = self._np_letters_lower[rnds[:, rnd_offset]]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(_LETTERS_LOWER[self._getRandomInt(0, 25, rndGenerator)])
-            elif char == 'A' and (not escape) ^ escapeSpecialMeaning:
-                placeholders[:, num_placeholders] = self._np_letters_upper[rnds[:, rnd_offset]]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(_LETTERS_UPPER[self._getRandomInt(0, 25, rndGenerator)])
-            elif char == 'k' and (not escape) ^ escapeSpecialMeaning:
-                placeholders[:, num_placeholders] = self._np_alnum_lower[rnds[:, rnd_offset]]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(_ALNUM_LOWER[self._getRandomInt(0, 35, rndGenerator)])
-            elif char == 'K' and (not escape) ^ escapeSpecialMeaning:
-                placeholders[:, num_placeholders] = self._np_alnum_upper[rnds[:, rnd_offset]]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(_ALNUM_UPPER[self._getRandomInt(0, 35, rndGenerator)])
-            elif char == 'n' and escape:
-                placeholders[:, num_placeholders] = rnds[:, rnd_offset]
+            elif char in escapedKeys and escape:
+                bound, valueMappings = self._templateEscapedMappings[char]
+
+                if valueMappings is not None:
+                    if selected_col0 is not None:
+                        placeholders[selected_col0, num_placeholders] = valueMappings[rnds[selected_col0, rnd_offset]]
+                    else:
+                        placeholders[:, num_placeholders] = valueMappings[rnds[:, rnd_offset]]
+                else:
+                    if selected_col0 is not None:
+                        placeholders[selected_col0, num_placeholders] = rnds[selected_col0, rnd_offset]
+                    else:
+                        placeholders[:, num_placeholders] = rnds[:, rnd_offset]
                 num_placeholders += 1
                 rnd_offset = rnd_offset + 1
                 # retval.append(str(self._getRandomInt(0, 255, rndGenerator)))
                 escape = False
-            elif char == 'N' and escape:
-                placeholders[:, num_placeholders] = rnds[:, rnd_offset]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(str(self._getRandomInt(0, 65535, rndGenerator)))
-                escape = False
-            elif char == 'W' and escape:
-                placeholders[:, num_placeholders] = self._upperWordList[rnds[:, rnd_offset]]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(self._upperWordList[self._getRandomWordOffset(self._lenWords, rndGenerator=rndGenerator)])
-                escape = False
-            elif char == 'w' and escape:
-                placeholders[:, num_placeholders] = self._wordList[rnds[:, rnd_offset]]
-                num_placeholders += 1
-                rnd_offset = rnd_offset + 1
-                # retval.append(self._wordList[self._getRandomWordOffset(self._lenWords, rndGenerator=rndGenerator)])
-                escape = False
             elif char == 'v' and escape:
                 escape = False
                 if following_char is not None and ('0' <= following_char <= '9'):
diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
index 0968724b..0ada9ede 100644
--- a/tests/test_text_templates.py
+++ b/tests/test_text_templates.py
@@ -144,8 +144,11 @@ def test_masking(self, template_provided, escapeSpecial, useTemplateObject):
 
 
     @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
-                             [ (r'\\w \\w|\\w A. \\w', False, False),
+                             [ (r'\w aAdDkKxX \n \N \w', False, False),
+                               (r'\\w \\w|\\w A. \\w', False, False),
+                               (r'\\w A. \\w|\\w \\w', False, False),
                                (r'\w \w|\w A. \w', False, False),
+                               (r'\w \w|\w \v. \w', False, False),
                                (r'\w A. \w', False, False),
                                (r'\\w \\w|\\w a. \\w', False, False),
                                 (r'\\w \\w|\\w k. \\w', False, False),
@@ -175,7 +178,7 @@ def test_masking(self, template_provided, escapeSpecial, useTemplateObject):
 
                                ])
 
-    def test_text_templates1(self, template_provided, escapeSpecial, useTemplateObject):
+    def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
         template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
         print(f"template [{template_provided}]")
 
@@ -216,7 +219,9 @@ def test_text_templates1(self, template_provided, escapeSpecial, useTemplateObje
 
 
     @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
-                             [ (r'\\w \\w|\\w A. \\w', False, False),
+                             [ (r'\w aAdDkK \w', False, False),
+
+                               (r'\\w \\w|\\w A. \\w', False, False),
                                (r'\w \w|\w A. \w', False, False),
                                (r'\w A. \w', False, False),
                                (r'\\w \\w|\\w a. \\w', False, False),
@@ -249,7 +254,7 @@ def test_text_templates1(self, template_provided, escapeSpecial, useTemplateObje
 
                                ])
 
-    def test_text_templates2(self, template_provided, escapeSpecial, useTemplateObject):
+    def test_full_build(self, template_provided, escapeSpecial, useTemplateObject):
         import dbldatagen as dg
         print(f"template [{template_provided}]")
 

From 80ffff9fbb20b306461389a2021cbd86d3128e27 Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Sun, 19 Mar 2023 20:29:38 -0700
Subject: [PATCH 04/12] wip

---
 tests/test_text_templates.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
index 0ada9ede..a6eae0e9 100644
--- a/tests/test_text_templates.py
+++ b/tests/test_text_templates.py
@@ -148,7 +148,7 @@ def test_masking(self, template_provided, escapeSpecial, useTemplateObject):
                                (r'\\w \\w|\\w A. \\w', False, False),
                                (r'\\w A. \\w|\\w \\w', False, False),
                                (r'\w \w|\w A. \w', False, False),
-                               (r'\w \w|\w \v. \w', False, False),
+                               #(r'\w \w|\w \v. \w', False, False),
                                (r'\w A. \w', False, False),
                                (r'\\w \\w|\\w a. \\w', False, False),
                                 (r'\\w \\w|\\w k. \\w', False, False),

From 612d6a1a9bcd74fdb647a92c0604332d5b2efcca Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Sun, 19 Mar 2023 21:22:48 -0700
Subject: [PATCH 05/12] wip

---
 dbldatagen/text_generators.py |  2 +-
 tests/test_text_templates.py  | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/dbldatagen/text_generators.py b/dbldatagen/text_generators.py
index 404f7ea8..7063df8b 100644
--- a/dbldatagen/text_generators.py
+++ b/dbldatagen/text_generators.py
@@ -444,7 +444,7 @@ def _get_values_subelement(elem):
 
         if isinstance(placeholders, np.ma.MaskedArray):
             selected = ~placeholders.mask
-            selected_col0 = selected[:, 1]
+            selected_col0 = selected[:, 0]
 
         # in the following code, the construct `(not escape) ^ self._escapeSpecialMeaning` means apply
         # special meaning if either escape is not true or the option `self._escapeSpecialMeaning` is true.
diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
index a6eae0e9..dd94ea23 100644
--- a/tests/test_text_templates.py
+++ b/tests/test_text_templates.py
@@ -175,7 +175,12 @@ def test_masking(self, template_provided, escapeSpecial, useTemplateObject):
                                (r'\\w k. \\w', False, True),
                                (r'\w \a. \w', True, True),
                                (r'\w \k. \w', True, True),
-
+                               (r'\w', True, True),
+                               (r'\w', False, True),
+                               (r'\w', False, False),
+                               (r'b', True, True),
+                               (r'b', False, True),
+                               (r'b', False, False),
                                ])
 
     def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
@@ -251,6 +256,9 @@ def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
                                (r'\w \k. \w', True, True),
                                (r'\w \w|\w \w \w|\w \n \w|\w \w \w \w', True, True),
                                (r'\w \n \w', True, True),
+                               (r'\w', True, True),
+                               (r'\w', False, True),
+                               (r'\w', False, False),
 
                                ])
 

From f181df6dc1873c2a3e7997e726087f5382f67697 Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Mon, 20 Mar 2023 18:53:38 -0700
Subject: [PATCH 06/12] wip

---
 dbldatagen/text_generators.py |  24 ++++++--
 tests/test_text_templates.py  | 113 +++++++++++++++++++++++++---------
 2 files changed, 105 insertions(+), 32 deletions(-)

diff --git a/dbldatagen/text_generators.py b/dbldatagen/text_generators.py
index 7063df8b..b51e655c 100644
--- a/dbldatagen/text_generators.py
+++ b/dbldatagen/text_generators.py
@@ -227,8 +227,7 @@ def __init__(self, template, escapeSpecialChars=False, extendedWordList=None):
 
         self._template = template
         self._escapeSpecialMeaning = bool(escapeSpecialChars)
-        template_str0 = self._template
-        self._templates = [x.replace('$__sep__', '|') for x in template_str0.replace(r'\|', '$__sep__').split('|')]
+        self._templates = self._splitTemplates(self._template)
         self._wordList = np.array(extendedWordList if extendedWordList is not None else _WORDS_LOWER)
         self._upperWordList = np.array([x.upper() for x in extendedWordList]
                                        if extendedWordList is not None else _WORDS_UPPER)
@@ -279,10 +278,21 @@ def __init__(self, template, escapeSpecialChars=False, extendedWordList=None):
         self._placeholders_needed = [ x[0] for x in template_info]
         self._template_rnd_bounds = [ x[1] for x in template_info]
 
-
     def __repr__(self):
         return f"TemplateGenerator(template='{self._template}')"
 
+    def _splitTemplates(self, templateStr):
+        """ Split template string into individual template strings
+
+        :param templateStr: template string
+        :return: list of individual template strings
+
+
+        """
+        tmp_template = templateStr.replace(r'\\', '$__escape__').replace(r'\|', '$__sep__')
+        results = [x.replace('$__escape__', r'\\').replace('$__sep__', '|') for x in tmp_template.split('|')]
+        return results
+
     @property
     def templates(self):
         """ Get effective templates for text generator"""
@@ -398,7 +408,13 @@ def _applyTemplateStringsForTemplate(self, baseValue, genTemplate, placeholders,
         `_escapeSpecialMeaning` parameter allows for backwards compatibility with old style syntax while allowing
         for preferred new style template syntax. Specify as True to force escapes for special meanings,.
 
-        Note that both placeholders and rnds are masked arrays
+        .. note::
+                Both `placeholders` and `rnds` are numpy masked arrays. If there are multiple templates in the template
+                generation source template, then this method will be called multiple times with each of
+                the distinct templates passed and the `placeholders` and `rnds` arrays masked so that the each call
+                will apply the template to rows to which that template applies.
+
+                The template may be the empty string.
 
         """
         assert baseValue.shape[0] == placeholders.shape[0]
diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
index dd94ea23..5635b92d 100644
--- a/tests/test_text_templates.py
+++ b/tests/test_text_templates.py
@@ -26,37 +26,91 @@ class TestTextTemplates:
     row_count = 100000
     partitions_requested = 4
 
+    @pytest.mark.parametrize("templates, splitTemplates",
+                               [
+                                   (r"a|b", ['a', 'b']),
+                                   (r"a|b|", ['a', 'b', '']),
+                                   (r"a", ['a']),
+                                   (r"", ['']),
+                                   (r"a\|b", [r'a|b']),
+                                   (r"a\\|b", [r'a\\', 'b']),
+                                   (r"a\|b|c", [r'a|b', 'c']),
+                                   (r"123,$456|test test2 |\|\a\\a |021 \| 123",
+                                    ['123,$456', 'test test2 ', '|\\a\\\\a ', '021 | 123']),
+                                   (
+                                   r"123 \\|  123 \|123 | 123|123|123 |asd023,\|23|",
+                                   ['123 \\\\', '  123 |123 ', ' 123', '123', '123 ', 'asd023,|23', '']),
+                                   (r" 123|123|123 |asd023,\|23", [' 123', '123', '123 ', 'asd023,|23']),
+                                   (r'',[ ''])
+                               ])
+    def test_split_templates(self, templates, splitTemplates):
+        tg1 = TemplateGenerator("test", escapeSpecialChars=False)
+
+        results = tg1._splitTemplates(templates)
+
+        assert results == splitTemplates
+
+
     @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
-                             [ (r'\\w \\w|\\w A. \\w', False, False),
-                               (r'\w \w|\w A. \w', False, False),
-                               (r'\w A. \w', False, False),
-                               (r'\\w \\w|\\w a. \\w', False, False),
-                                (r'\\w \\w|\\w k. \\w', False, False),
+                             [ #(r'\w \w|\w \v. \w', False, False),
+                                (r'A', False, True),
+                                (r'D', False, True),
+                                (r'K', False, True),
+                                (r'X', False, True),
+                                (r'\W', False, True),
+                                (r'\W', True, True),
+                                (r'\\w A. \\w|\\w \\w', False, False),
+                                (r'\\w \\w|\\w A. \\w', False, False),
+                                (r'\\w \\w|\\w A. \\w', False, True),
+                                (r'\\w \\w|\\w A. \\w', True, True),
+                                (r'\\w \\w|\\w A. \\w|\w n n \w', False, False),
                                 (r'\\w \\w|\\w K. \\w', False, False),
-                                (r'\\w \\w|\\w x. \\w', False, False),
+                                (r'\\w \\w|\\w K. \\w', False, True),
+                                (r'\\w \\w|\\w K. \\w', True, True),
                                 (r'\\w \\w|\\w X. \\w', False, False),
-                               (r'\\w \\w|\\w A. \\w', False, True),
-                               (r'\\w \\w|\\w a. \\w', False, True),
-                               (r'\\w \\w|\\w k. \\w', False, True),
-                               (r'\\w \\w|\\w K. \\w', False, True),
-                               (r'\\w \\w|\\w x. \\w', False, True),
-                               (r'\\w \\w|\\w X. \\w', False, True),
-                               (r'\\w \\w|\\w A. \\w', True, True),
-                               (r'\w \w|\w \A. \w', True, True),
-                               (r'\\w \\w|\\w a. \\w', True, True),
-                               (r'\w \w|\w \a. \w', True, True),
-                               (r'\\w \\w|\\w k. \\w', True, True),
-                               (r'\\w \\w|\\w K. \\w', True, True),
-                               (r'\\w \\w|\\w x. \\w', True, True),
-                               (r'\\w \\w|\\w X. \\w', True, True),
-                               (r'\\w a. \\w|\\w \\w', False, False),
-                               (r'\\w k. \\w|\\w \\w', False, False),
-                               (r'\\w a. \\w', False, True),
-                               (r'\\w k. \\w', False, True),
-                               (r'\w \a. \w', True, True),
-                               (r'\w \k. \w', True, True),
-
-                               ])
+                                (r'\\w \\w|\\w X. \\w', False, True),
+                                (r'\\w \\w|\\w X. \\w', True, True),
+                                (r'\\w \\w|\\w a. \\w', False, False),
+                                (r'\\w \\w|\\w a. \\w', False, True),
+                                (r'\\w \\w|\\w a. \\w', True, True),
+                                (r'\\w \\w|\\w k. \\w', False, False),
+                                (r'\\w \\w|\\w k. \\w', False, True),
+                                (r'\\w \\w|\\w k. \\w', True, True),
+                                (r'\\w \\w|\\w x. \\w', False, False),
+                                (r'\\w \\w|\\w x. \\w', False, True),
+                                (r'\\w \\w|\\w x. \\w', True, True),
+                                (r'\\w a. \\w', False, True),
+                                (r'\\w a. \\w|\\w \\w', False, False),
+                                (r'\\w k. \\w', False, True),
+                                (r'\\w k. \\w|\\w \\w', False, False),
+                                (r'\n', False, True),
+                                (r'\n', True, True),
+                                (r'\v', False, True),
+                                (r'\v', True, True),
+                                (r'\w A. \w', False, False),
+                                (r'\w \a. \w', True, True),
+                                (r'\w \k. \w', True, True),
+                                (r'\w \n \w', True, True),
+                                (r'\w \w|\w A. \w', False, False),
+                                (r'\w \w|\w \A. \w', True, True),
+                                (r'\w \w|\w \a. \w', True, True),
+                                (r'\w \w|\w \w \w|\w \n \w|\w \w \w \w', True, True),
+                                (r'\w aAdDkK \w', False, False),
+                                (r'\w aAdDkKxX \n \N \w', False, False),
+                                (r'\w', False, False),
+                                (r'\w', False, True),
+                                (r'\w', True, True),
+                                (r'a', False, True),
+                                (r'b', False, False),
+                                (r'b', False, True),
+                                (r'b', True, True),
+                                (r'd', False, True),
+                                (r'k', False, True),
+                                (r'x', False, True),
+                                 ('', False, True),
+                                 ('', False, True),
+                                 (r'', False, True),
+                             ])
 
     def test_rnd_compute(self, template_provided, escapeSpecial, useTemplateObject):
         template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
@@ -66,6 +120,9 @@ def test_rnd_compute(self, template_provided, escapeSpecial, useTemplateObject):
 
         template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
 
+        assert template_choices is not None
+        assert template_rnd_bounds is not None
+        assert template_rnds is not None
         assert len(template_choices) == len(template_rnds)
         assert len(template_choices) == len(template_rnd_bounds)
 

From cf2b440e8c2f271adeb2feb6592c36786a067873 Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Mon, 20 Mar 2023 19:56:33 -0700
Subject: [PATCH 07/12] wip

---
 dbldatagen/text_generators.py | 55 ++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 20 deletions(-)

diff --git a/dbldatagen/text_generators.py b/dbldatagen/text_generators.py
index b51e655c..578aae40 100644
--- a/dbldatagen/text_generators.py
+++ b/dbldatagen/text_generators.py
@@ -243,6 +243,7 @@ def __init__(self, template, escapeSpecialChars=False, extendedWordList=None):
         self._np_letters_all = np.array(_LETTERS_ALL)
         self._lenWords = len(self._wordList)
 
+        # mappings must be mapping from string to tuple(length of mappings, mapping array or list)
         self._templateMappings = {
             'a': (26, self._np_letters_lower),
             'A': (26, self._np_letters_upper),
@@ -254,6 +255,17 @@ def __init__(self, template, escapeSpecialChars=False, extendedWordList=None):
             'K': (36, self._np_alnum_upper)
         }
 
+        # ensure that each mapping is mapping from string to list or numpy array
+        for k, v in self._templateMappings.items():
+            assert (k is not None) and isinstance(k, str) and len(k) > 0, "key must be non-empty string"
+            assert v is not None and isinstance(v, tuple) and len(v) == 2, "value must be tuple of length 2"
+            mapping_length, mappings = v
+            print(type(mappings))
+            assert isinstance(mapping_length, int), "mapping length must be of type int"
+            assert isinstance(mappings, list) or isinstance(mappings,  np.ndarray),\
+                "mappings are lists or numpy arrays"
+            assert mapping_length == 0 or len(mappings) == mapping_length, "mappings must match mapping_length"
+
         self._templateEscapedMappings = {
             'n': (256, None),
             'N': (65536, None),
@@ -261,6 +273,19 @@ def __init__(self, template, escapeSpecialChars=False, extendedWordList=None):
             'W': (self._lenWords, self._upperWordList)
         }
 
+        # ensure that each escaped mapping is mapping from string to None, list or numpy array
+        for k, v in self._templateEscapedMappings.items():
+            assert (k is not None) and isinstance(k, str) and len(k) > 0, "key must be non-empty string"
+            assert v is not None and isinstance(v, tuple) and len(v) == 2, "value must be tuple of length 2"
+            mapping_length, mappings = v
+            assert isinstance(mapping_length, int), "mapping length must be of type int"
+            assert mappings is None or isinstance(mappings, list) or isinstance(mappings,  np.ndarray),\
+                "mappings are lists or numpy arrays"
+
+            # for escaped mappings, the mapping can be None in which case the mapping is to the number itself
+            # i.e mapping[4] = 4
+            assert mappings is None or len(mappings) == mapping_length, "mappings must match mapping_length"
+
         # get the template metadata - this will be list of metadata entries for each template
         # for each template, metadata will be tuple of number of placeholders followed by list of random bounds
         # to be computed when replacing non static placeholder
@@ -453,14 +478,13 @@ def _get_values_subelement(elem):
         num_placeholders = 0
         rnd_offset = 0
 
-        selected = None
-        selected_col0 = None
+        masked_rows = None
 
         assert isinstance(placeholders, np.ma.MaskedArray), "expecting MaskArray"
 
         if isinstance(placeholders, np.ma.MaskedArray):
-            selected = ~placeholders.mask
-            selected_col0 = selected[:, 0]
+            active_rows = ~placeholders.mask
+            masked_rows = active_rows[:, 0]
 
         # in the following code, the construct `(not escape) ^ self._escapeSpecialMeaning` means apply
         # special meaning if either escape is not true or the option `self._escapeSpecialMeaning` is true.
@@ -482,19 +506,10 @@ def _get_values_subelement(elem):
                 # random numbers from `rnds` 2d array
                 bound, valueMappings = self._templateMappings[char]
 
-                print(f"mappings - {bound},{char}", valueMappings)
-                print(f"selected_col0 - {selected_col0}")
-
-                if valueMappings is not None:
-                    if selected_col0 is not None:
-                        placeholders[selected_col0, num_placeholders] = valueMappings[rnds[selected_col0, rnd_offset]]
-                    else:
-                        placeholders[:, num_placeholders] = valueMappings[rnds[:, rnd_offset]]
+                if masked_rows is not None:
+                    placeholders[masked_rows, num_placeholders] = valueMappings[rnds[masked_rows, rnd_offset]]
                 else:
-                    if selected_col0 is not None:
-                        placeholders[selected_col0, num_placeholders] = rnds[selected_col0, rnd_offset]
-                    else:
-                        placeholders[:, num_placeholders] = rnds[:, rnd_offset]
+                    placeholders[:, num_placeholders] = valueMappings[rnds[:, rnd_offset]]
 
                 num_placeholders += 1
                 rnd_offset = rnd_offset + 1
@@ -504,13 +519,13 @@ def _get_values_subelement(elem):
                 bound, valueMappings = self._templateEscapedMappings[char]
 
                 if valueMappings is not None:
-                    if selected_col0 is not None:
-                        placeholders[selected_col0, num_placeholders] = valueMappings[rnds[selected_col0, rnd_offset]]
+                    if masked_rows is not None:
+                        placeholders[masked_rows, num_placeholders] = valueMappings[rnds[masked_rows, rnd_offset]]
                     else:
                         placeholders[:, num_placeholders] = valueMappings[rnds[:, rnd_offset]]
                 else:
-                    if selected_col0 is not None:
-                        placeholders[selected_col0, num_placeholders] = rnds[selected_col0, rnd_offset]
+                    if masked_rows is not None:
+                        placeholders[masked_rows, num_placeholders] = rnds[masked_rows, rnd_offset]
                     else:
                         placeholders[:, num_placeholders] = rnds[:, rnd_offset]
                 num_placeholders += 1

From 5d97299d1ae70a652e4c47fb8595cef9e87f89a1 Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Mon, 20 Mar 2023 20:55:38 -0700
Subject: [PATCH 08/12] wip

---
 docs/source/APIDOCS.md              |  14 +--
 docs/source/generating_cdc_data.rst |  10 +-
 docs/source/multi_table_data.rst    |   2 +-
 docs/source/textdata.rst            |   5 +-
 tests/test_text_templates.py        | 174 +++++++++++-----------------
 5 files changed, 84 insertions(+), 121 deletions(-)

diff --git a/docs/source/APIDOCS.md b/docs/source/APIDOCS.md
index c786458a..10679103 100644
--- a/docs/source/APIDOCS.md
+++ b/docs/source/APIDOCS.md
@@ -250,11 +250,11 @@ dataspec = (dg.DataGenerator(spark, rows=10000000, partitions=8,
             .withSchema(table_schema))
 
 dataspec = (dataspec
-                .withColumnSpec("name", percentNulls=0.01, template=r'\\w \\w|\\w a. \\w')                                       
+                .withColumnSpec("name", percentNulls=0.01, template=r'\w \w|\w a. \w')                                       
                 .withColumnSpec("serial_number", minValue=1000000, maxValue=10000000, 
                                  prefix="dr", random=True) 
-                .withColumnSpec("email", template=r'\\w.\\w@\\w.com')       
-                .withColumnSpec("license_plate", template=r'\\n-\\n')
+                .withColumnSpec("email", template=r'\w.\w@\w.com')       
+                .withColumnSpec("license_plate", template=r'\n-\n')
            )
 df1 = dataspec.build()
 
@@ -472,7 +472,7 @@ data_rows = 10000000
 spark.conf.set("spark.sql.shuffle.partitions", shuffle_partitions_requested)
 
 dataspec = (dg.DataGenerator(spark, rows=data_rows, partitions=8, randomSeedMethod="hash_fieldname")
-                .withColumn("name", percentNulls=0.01, template=r'\\w \\w|\\w a. \\w') 
+                .withColumn("name", percentNulls=0.01, template=r'\w \w|\w a. \w') 
                 .withColumn("payment_instrument_type", values=['paypal', 'visa', 'mastercard', 'amex'], 
                             random=True)             
                 .withColumn("int_payment_instrument", "int",  minValue=0000, maxValue=9999,  
@@ -481,7 +481,7 @@ dataspec = (dg.DataGenerator(spark, rows=data_rows, partitions=8, randomSeedMeth
                 .withColumn("payment_instrument", 
                              expr="format_number(int_payment_instrument, '**** ****** *####')",
                              baseColumn="int_payment_instrument")
-                .withColumn("email", template=r'\\w.\\w@\\w.com')       
+                .withColumn("email", template=r'\w.\w@\w.com')       
                 .withColumn("md5_payment_instrument", 
                             expr="md5(concat(payment_instrument_type, ':', payment_instrument))",
                             baseColumn=['payment_instrument_type', 'payment_instrument']) 
@@ -524,7 +524,7 @@ spark.conf.set("spark.sql.shuffle.partitions", shuffle_partitions_requested)
 dataspec = (
     dg.DataGenerator(spark, rows=data_rows, partitions=8, randomSeedMethod="hash_fieldname", 
                      randomSeed=42)
-    .withColumn("name", percentNulls=0.01, template=r'\\w \\w|\\w a. \\w')
+    .withColumn("name", percentNulls=0.01, template=r'\w \w|\w a. \w')
     .withColumn("payment_instrument_type", values=['paypal', 'visa', 'mastercard', 'amex'],
                 random=True)
     .withColumn("int_payment_instrument", "int",  minValue=0000, maxValue=9999,  
@@ -533,7 +533,7 @@ dataspec = (
     .withColumn("payment_instrument", 
                 expr="format_number(int_payment_instrument, '**** ****** *####')",
                 baseColumn="int_payment_instrument")
-    .withColumn("email", template=r'\\w.\\w@\\w.com')
+    .withColumn("email", template=r'\w.\w@\w.com')
     .withColumn("md5_payment_instrument",
                 expr="md5(concat(payment_instrument_type, ':', payment_instrument))",
                 baseColumn=['payment_instrument_type', 'payment_instrument'])
diff --git a/docs/source/generating_cdc_data.rst b/docs/source/generating_cdc_data.rst
index 6528f4df..a3750690 100644
--- a/docs/source/generating_cdc_data.rst
+++ b/docs/source/generating_cdc_data.rst
@@ -49,8 +49,8 @@ We'll add a timestamp for when the row was generated and a memo field to mark wh
 
    dataspec = (dg.DataGenerator(spark, rows=data_rows, partitions=partitions_requested)
                .withColumn("customer_id","long", uniqueValues=uniqueCustomers)
-               .withColumn("name", percentNulls=0.01, template=r'\\w \\w|\\w a. \\w')
-               .withColumn("alias", percentNulls=0.01, template=r'\\w \\w|\\w a. \\w')
+               .withColumn("name", percentNulls=0.01, template=r'\w \w|\w a. \w')
+               .withColumn("alias", percentNulls=0.01, template=r'\w \w|\w a. \w')
                .withColumn("payment_instrument_type", values=['paypal', 'Visa', 'Mastercard',
                            'American Express', 'discover', 'branded visa', 'branded mastercard'],
                            random=True, distribution="normal")
@@ -58,9 +58,9 @@ We'll add a timestamp for when the row was generated and a memo field to mark wh
                            baseColumnType="hash", omit=True)
                .withColumn("payment_instrument", expr="format_number(int_payment_instrument, '**** ****** *####')",
                            baseColumn="int_payment_instrument")
-               .withColumn("email", template=r'\\w.\\w@\\w.com|\\w-\\w@\\w')
-               .withColumn("email2", template=r'\\w.\\w@\\w.com')
-               .withColumn("ip_address", template=r'\\n.\\n.\\n.\\n')
+               .withColumn("email", template=r'\w.\w@\w.com|\w-\w@\w')
+               .withColumn("email2", template=r'\w.\w@\w.com')
+               .withColumn("ip_address", template=r'\n.\n.\n.\n')
                .withColumn("md5_payment_instrument",
                            expr="md5(concat(payment_instrument_type, ':', payment_instrument))",
                            base_column=['payment_instrument_type', 'payment_instrument'])
diff --git a/docs/source/multi_table_data.rst b/docs/source/multi_table_data.rst
index 44f99150..a3e4e988 100644
--- a/docs/source/multi_table_data.rst
+++ b/docs/source/multi_table_data.rst
@@ -167,7 +167,7 @@ when using hashed values, the range of the hashes produced can be large.
    customer_dataspec = (dg.DataGenerator(spark, rows=data_rows, partitions=partitions_requested)
                .withColumn("customer_id","decimal(10)", minValue=CUSTOMER_MIN_VALUE,
                            uniqueValues=UNIQUE_CUSTOMERS)
-               .withColumn("customer_name", template=r"\\w \\w|\\w a. \\w")
+               .withColumn("customer_name", template=r"\w \w|\w a. \w")
 
                # use the following for a simple sequence
                #.withColumn("device_id","decimal(10)", minValue=DEVICE_MIN_VALUE,
diff --git a/docs/source/textdata.rst b/docs/source/textdata.rst
index 8b249783..9393da2b 100644
--- a/docs/source/textdata.rst
+++ b/docs/source/textdata.rst
@@ -164,6 +164,9 @@ If set to False, then the template ``r"\\dr_\\v"`` will generate the values ``"d
 to the values zero to 999. This conforms to earlier implementations for backwards compatibility.
 
 If set to True, then the template ``r"dr_\\v"`` will generate the values ``"dr_0"`` ... ``"dr_999"``
-when applied to the values zero to 999. This conforms to the preferred style going forward
+when applied to the values zero to 999. This conforms to the preferred style going forward. In other words the char `d`
+will not be treated as a special char.
+
+.. note::
 
 
diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
index 5635b92d..c7b15217 100644
--- a/tests/test_text_templates.py
+++ b/tests/test_text_templates.py
@@ -107,9 +107,9 @@ def test_split_templates(self, templates, splitTemplates):
                                 (r'd', False, True),
                                 (r'k', False, True),
                                 (r'x', False, True),
+                                 ('', False, False),
                                  ('', False, True),
-                                 ('', False, True),
-                                 (r'', False, True),
+                                 (r'', True, True),
                              ])
 
     def test_rnd_compute(self, template_provided, escapeSpecial, useTemplateObject):
@@ -136,110 +136,74 @@ def test_rnd_compute(self, template_provided, escapeSpecial, useTemplateObject):
                 assert bounds[iy] == -1 or (rnds[iy] < bounds[iy])
 
     @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
-                             [ (r'\\w \\w|\\w A. \\w|\w n n \w', False, False),
-                               (r'\w \w|\w A. \w', False, False),
-                               (r'\w A. \w', False, False),
-                               (r'\\w \\w|\\w a. \\w', False, False),
-                                (r'\\w \\w|\\w k. \\w', False, False),
+                             [ #(r'\w \w|\w \v. \w', False, False),
+                                 (r'\\w \\w|\\w a. \\w', False, False),
+                                 (r'\\w \\w|\\w a. \\w', False, True),
+                                 (r'\\w \\w|\\w a. \\w', True, True),
+                                 (r'\w \w|\w a. \w', False, False),
+                                 (r'\w.\w@\w.com', False, False),
+                                 (r'\n-\n', False, False),
+                                 (r'A', False, True),
+                                (r'D', False, True),
+                                (r'K', False, True),
+                                (r'X', False, True),
+                                (r'\W', False, True),
+                                (r'\W', True, True),
+                                (r'\\w A. \\w|\\w \\w', False, False),
+                                (r'\\w \\w|\\w A. \\w', False, False),
+                                (r'\\w \\w|\\w A. \\w', False, True),
+                                (r'\\w \\w|\\w A. \\w', True, True),
+                                (r'\\w \\w|\\w A. \\w|\w n n \w', False, False),
                                 (r'\\w \\w|\\w K. \\w', False, False),
-                                (r'\\w \\w|\\w x. \\w', False, False),
+                                (r'\\w \\w|\\w K. \\w', False, True),
+                                (r'\\w \\w|\\w K. \\w', True, True),
                                 (r'\\w \\w|\\w X. \\w', False, False),
-                               (r'\\w \\w|\\w A. \\w', False, True),
-                               (r'\\w \\w|\\w a. \\w', False, True),
-                               (r'\\w \\w|\\w k. \\w', False, True),
-                               (r'\\w \\w|\\w K. \\w', False, True),
-                               (r'\\w \\w|\\w x. \\w', False, True),
-                               (r'\\w \\w|\\w X. \\w', False, True),
-                               (r'\\w \\w|\\w A. \\w', True, True),
-                               (r'\w \w|\w \A. \w', True, True),
-                               (r'\\w \\w|\\w a. \\w', True, True),
-                               (r'\w \w|\w \a. \w', True, True),
-                               (r'\\w \\w|\\w k. \\w', True, True),
-                               (r'\\w \\w|\\w K. \\w', True, True),
-                               (r'\\w \\w|\\w x. \\w', True, True),
-                               (r'\\w \\w|\\w X. \\w', True, True),
-                               (r'\\w a. \\w|\\w \\w', False, False),
-                               (r'\\w k. \\w|\\w \\w', False, False),
-                               (r'\\w a. \\w', False, True),
-                               (r'\\w k. \\w', False, True),
-                               (r'\w \a. \w', True, True),
-                               (r'\w \k. \w', True, True),
-
-                               ])
-
-    def test_masking(self, template_provided, escapeSpecial, useTemplateObject):
-        template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
-        print(f"template [{template_provided}]")
-
-        arr = np.arange(100)
-
-        template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
-
-        assert len(template_choices) == len(template_rnds)
-        assert len(template_choices) == len(template_rnd_bounds)
-
-        arr2 = np.full(shape=(arr.shape[0], 2), fill_value="testing 1 2 3 4 ", dtype=np.object_)
-        print(template_choices)
-
-        template_choices_t = template_choices.T
-
-        masked_placeholders = np.ma.MaskedArray(arr2, mask=False)
-        for x in range(len(template1._templates)-1):
-            masked_placeholders[template_choices_t != x] = np.ma.masked
-            np.ma.harden_mask(masked_placeholders)
-
-            masked_placeholders[:, 0] = template1.templates[x]
-            masked_placeholders[:, 1] = template1.templates[x]
-
-            np.ma.soften_mask(masked_placeholders)
-            masked_placeholders.mask = False
-
-
-        print(arr2)
-
-
-
-
-    @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
-                             [ (r'\w aAdDkKxX \n \N \w', False, False),
-                               (r'\\w \\w|\\w A. \\w', False, False),
-                               (r'\\w A. \\w|\\w \\w', False, False),
-                               (r'\w \w|\w A. \w', False, False),
-                               #(r'\w \w|\w \v. \w', False, False),
-                               (r'\w A. \w', False, False),
-                               (r'\\w \\w|\\w a. \\w', False, False),
+                                (r'\\w \\w|\\w X. \\w', False, True),
+                                (r'\\w \\w|\\w X. \\w', True, True),
+                                (r'\\w \\w|\\w a. \\w', False, False),
+                                (r'\\w \\w|\\w a. \\w', False, True),
+                                (r'\\w \\w|\\w a. \\w', True, True),
                                 (r'\\w \\w|\\w k. \\w', False, False),
-                                (r'\\w \\w|\\w K. \\w', False, False),
+                                (r'\\w \\w|\\w k. \\w', False, True),
+                                (r'\\w \\w|\\w k. \\w', True, True),
                                 (r'\\w \\w|\\w x. \\w', False, False),
-                                (r'\\w \\w|\\w X. \\w', False, False),
-                               (r'\\w \\w|\\w A. \\w', False, True),
-                               (r'\\w \\w|\\w a. \\w', False, True),
-                               (r'\\w \\w|\\w k. \\w', False, True),
-                               (r'\\w \\w|\\w K. \\w', False, True),
-                               (r'\\w \\w|\\w x. \\w', False, True),
-                               (r'\\w \\w|\\w X. \\w', False, True),
-                               (r'\\w \\w|\\w A. \\w', True, True),
-                               (r'\w \w|\w \A. \w', True, True),
-                               (r'\\w \\w|\\w a. \\w', True, True),
-                               (r'\w \w|\w \a. \w', True, True),
-                               (r'\\w \\w|\\w k. \\w', True, True),
-                               (r'\\w \\w|\\w K. \\w', True, True),
-                               (r'\\w \\w|\\w x. \\w', True, True),
-                               (r'\\w \\w|\\w X. \\w', True, True),
-                               (r'\\w a. \\w|\\w \\w', False, False),
-                               (r'\\w k. \\w|\\w \\w', False, False),
-                               (r'\\w a. \\w', False, True),
-                               (r'\\w k. \\w', False, True),
-                               (r'\w \a. \w', True, True),
-                               (r'\w \k. \w', True, True),
-                               (r'\w', True, True),
-                               (r'\w', False, True),
-                               (r'\w', False, False),
-                               (r'b', True, True),
-                               (r'b', False, True),
-                               (r'b', False, False),
-                               ])
-
+                                (r'\\w \\w|\\w x. \\w', False, True),
+                                (r'\\w \\w|\\w x. \\w', True, True),
+                                (r'\\w a. \\w', False, True),
+                                (r'\\w a. \\w|\\w \\w', False, False),
+                                (r'\\w k. \\w', False, True),
+                                (r'\\w k. \\w|\\w \\w', False, False),
+                                (r'\n', False, True),
+                                (r'\n', True, True),
+                                (r'\v', False, True),
+                                (r'\v', True, True),
+                                (r'\w A. \w', False, False),
+                                (r'\w \a. \w', True, True),
+                                (r'\w \k. \w', True, True),
+                                (r'\w \n \w', True, True),
+                                (r'\w \w|\w A. \w', False, False),
+                                (r'\w \w|\w \A. \w', True, True),
+                                (r'\w \w|\w \a. \w', True, True),
+                                (r'\w \w|\w \w \w|\w \n \w|\w \w \w \w', True, True),
+                                (r'\w aAdDkK \w', False, False),
+                                (r'\w aAdDkKxX \n \N \w', False, False),
+                                (r'\w', False, False),
+                                (r'\w', False, True),
+                                (r'\w', True, True),
+                                (r'a', False, True),
+                                (r'b', False, False),
+                                (r'b', False, True),
+                                (r'b', True, True),
+                                (r'd', False, True),
+                                (r'k', False, True),
+                                (r'x', False, True),
+                                 ('', False, False),
+                                 ('', False, True),
+                                 (r'', True, True),
+                                 ('|', False, False),
+                                 ('|', False, True),
+                                 (r'|', True, True),
+                             ])
     def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
         template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
         print(f"template [{template_provided}]")
@@ -275,11 +239,6 @@ def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
         results = template1.pandasGenerateText(arr)
         print(results)
 
-
-
-
-
-
     @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
                              [ (r'\w aAdDkK \w', False, False),
 
@@ -320,6 +279,7 @@ def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
                                ])
 
     def test_full_build(self, template_provided, escapeSpecial, useTemplateObject):
+        pytest.skip("skipping to see if this is needed for coverage")
         import dbldatagen as dg
         print(f"template [{template_provided}]")
 

From 9908e39ad2f82b17bd6c3cbc736240344b9abb16 Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Mon, 20 Mar 2023 22:12:15 -0700
Subject: [PATCH 09/12] wip

---
 .github/workflows/push.yml    |  3 ++-
 dbldatagen/text_generators.py | 24 ++++++++++++++++++++----
 tests/test_text_templates.py  | 26 ++++++++++++++++++++++++--
 3 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index b9ac6080..f9cfc957 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -48,4 +48,5 @@ jobs:
         with:
           files: ./coverage.xml
           name: dbldatagen
-          verbose: true
\ No newline at end of file
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
\ No newline at end of file
diff --git a/dbldatagen/text_generators.py b/dbldatagen/text_generators.py
index 578aae40..7a56d80f 100644
--- a/dbldatagen/text_generators.py
+++ b/dbldatagen/text_generators.py
@@ -169,6 +169,7 @@ class TemplateGenerator(TextGenerator):  # lgtm [py/missing-equals]
     :param escapeSpecialChars: By default special chars in the template have special meaning if unescaped
                                If set to true, then the special meaning requires escape char ``\\``
     :param extendedWordList: if provided, use specified word list instead of default word list
+    :param legacyEscapeTreatment: if True, sequences of escape char are treated as single escape, Defaults to True
 
     The template generator generates text from a template to allow for generation of synthetic account card numbers,
     VINs, IBANs and many other structured codes.
@@ -219,14 +220,20 @@ class TemplateGenerator(TextGenerator):  # lgtm [py/missing-equals]
     If set to True, then the template ``r"dr_\\v"`` will generate the values ``"dr_0"`` ... ``"dr_999"``
     when applied to the values zero to 999. This conforms to the preferred style going forward
 
+    .. note::
+              In previous versions, multiple sequences of escape char are treated as single escape char
+              so template of r'\\w' and r'\w' have same meaning and dont emit escape char. If `legacyEscapeTreatment`
+              is false, r'\\w' will emit escape char in output string.
+
     """
 
-    def __init__(self, template, escapeSpecialChars=False, extendedWordList=None):
+    def __init__(self, template, escapeSpecialChars=False, extendedWordList=None, legacyEscapeTreatment=True):
         assert template is not None, "`template` must be specified"
         super().__init__()
 
         self._template = template
         self._escapeSpecialMeaning = bool(escapeSpecialChars)
+        self._legacyEscapeTreatment = legacyEscapeTreatment
         self._templates = self._splitTemplates(self._template)
         self._wordList = np.array(extendedWordList if extendedWordList is not None else _WORDS_LOWER)
         self._upperWordList = np.array([x.upper() for x in extendedWordList]
@@ -375,7 +382,10 @@ def _prepareTemplateStrings(self, genTemplate, escapeSpecialMeaning=False):
             char = genTemplate[i]
             following_char = genTemplate[i + 1] if i + 1 < template_len else None
 
-            if char == '\\':
+            if char == '\\' and (escape and not self._legacyEscapeTreatment):
+                escape = False
+                num_placeholders += 1
+            elif char == '\\':
                 escape = True
             elif use_value and ('0' <= char <= '9'):
                 # val_index = int(char)
@@ -482,7 +492,8 @@ def _get_values_subelement(elem):
 
         assert isinstance(placeholders, np.ma.MaskedArray), "expecting MaskArray"
 
-        if isinstance(placeholders, np.ma.MaskedArray):
+        # if template is empty, then nothing needs to be done
+        if template_len > 0 and isinstance(placeholders, np.ma.MaskedArray):
             active_rows = ~placeholders.mask
             masked_rows = active_rows[:, 0]
 
@@ -493,7 +504,12 @@ def _get_values_subelement(elem):
             char = genTemplate[i]
             following_char = genTemplate[i + 1] if i + 1 < template_len else None
 
-            if char == '\\':
+            if char == '\\' and (escape and not self._legacyEscapeTreatment):
+                escape = False
+                placeholders[:, num_placeholders] = char
+                # retval.append(char)
+                num_placeholders += 1
+            elif char == '\\':
                 escape = True
             elif use_value and ('0' <= char <= '9'):
                 val_index = int(char)
diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
index c7b15217..88299966 100644
--- a/tests/test_text_templates.py
+++ b/tests/test_text_templates.py
@@ -177,6 +177,10 @@ def test_rnd_compute(self, template_provided, escapeSpecial, useTemplateObject):
                                 (r'\n', True, True),
                                 (r'\v', False, True),
                                 (r'\v', True, True),
+                                (r'\v|\v-\v', False, True),
+                                (r'\v|\v-\v', True, True),
+                                (r'short string|a much longer string which is bigger than short string', False, True),
+                                (r'short string|a much longer string which is bigger than short string', True, True),
                                 (r'\w A. \w', False, False),
                                 (r'\w \a. \w', True, True),
                                 (r'\w \k. \w', True, True),
@@ -203,6 +207,8 @@ def test_rnd_compute(self, template_provided, escapeSpecial, useTemplateObject):
                                  ('|', False, False),
                                  ('|', False, True),
                                  (r'|', True, True),
+                                 (r'\ww - not e\xpecting two wor\ds', False, False),
+                                 (r'\ww - not expecting two words', True, True)
                              ])
     def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
         template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
@@ -215,7 +221,9 @@ def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
 
         print("templates", template1.templates)
 
-        arr = np.arange(100)
+        TEST_ROWS = 100
+
+        arr = np.arange(TEST_ROWS)
 
         template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
 
@@ -237,7 +245,21 @@ def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
 
 
         results = template1.pandasGenerateText(arr)
-        print(results)
+        assert results is not None
+
+        results_list = results.tolist()
+
+        results_rows = len(results_list)
+        assert results_rows == TEST_ROWS
+
+        for r in range(len(results)):
+            result_str = results[r]
+            assert result_str is not None and isinstance(result_str, str)
+            assert len(result_str) >= 0
+
+        print("results")
+        for i in range(len(results)):
+            print(f"{i}: '{results[i]}'")
 
     @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
                              [ (r'\w aAdDkK \w', False, False),

From 8b8856c42cced0a3c581a6da78c226d22ecc94e8 Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Mon, 20 Mar 2023 23:20:40 -0700
Subject: [PATCH 10/12] wip

---
 tests/test_text_templates.py | 158 +++++++++++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)

diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
index 88299966..40b2aecf 100644
--- a/tests/test_text_templates.py
+++ b/tests/test_text_templates.py
@@ -261,6 +261,164 @@ def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
         for i in range(len(results)):
             print(f"{i}: '{results[i]}'")
 
+    @pytest.mark.parametrize("templateProvided, escapeSpecial, legacyEscapeTreatment,expectedPattern ",
+                             [ (r'\\w \w', False, True, r"[a-z]+ [a-z]+"),
+                               (r'\\w \w', False, False, r"\\[a-z]+ [a-z]+"),
+                               (r'\\w \w', True, True, r"[a-z]+ [a-z]+"),
+                               (r'\\w \w', True, False, r"\\[a-z]+ [a-z]+"),
+                               (r'\n-\n', False, False, r"[0-9]+-[0-9]+"),
+                               (r'\\n-\n', False, True, r"[0-9]+-[0-9]+"),
+                               (r'\\n-\n', False, False, r"\\[0-9]+-[0-9]+"),
+                               (r'\\n-\n', True, True, r"[0-9]+-[0-9]+"),
+                               (r'\\n-\n', True, False, r"\\[0-9]+-[0-9]+"),
+                               (r'\\a', True, False, r"\\[a-z]"),
+                               (r'\\a', False, False, r"\\[a-z]"),
+                               (r'\\a', False, True, r"[a-z]"),
+                               (r'\\a', True, True, r"\\[a-z]"),
+                               ])
+    def test_escape_treatment(self, templateProvided, escapeSpecial, legacyEscapeTreatment, expectedPattern):
+
+        template1 = TemplateGenerator(templateProvided, escapeSpecialChars=escapeSpecial,
+                                      legacyEscapeTreatment=legacyEscapeTreatment)
+
+        TEST_ROWS = 100
+
+        arr = np.arange(TEST_ROWS)
+
+        results = template1.pandasGenerateText(arr)
+        assert results is not None
+
+        results_list = results.tolist()
+
+        results_rows = len(results_list)
+        assert results_rows == TEST_ROWS
+
+        print(f"expected pattern - '{expectedPattern}', template '{templateProvided}'" )
+        patt = re.compile(expectedPattern)
+
+        print("results")
+        for i in range(len(results)):
+            print(f"{i}: '{results[i]}'")
+            assert isinstance(results[i], str)
+            assert patt.match(results[i]) is not None, f"expecting match '{results[i]}' === '{expectedPattern}'"
+
+    @pytest.mark.parametrize("template_provided, escapeSpecial, legacyEscapeTreatment,expectedPattern ",
+                             [ (r'\\w \\w|\\w a. \\w', False, False, ""),
+                                 (r'\\w \\w|\\w a. \\w', False, True, ""),
+                                 (r'\w \w|\w a. \w', False, False, ""),
+                                 (r'\w.\w@\w.com', False, False, ""),
+                                 (r'\n-\n', False, False, ""),
+                             ])
+    def test_value_sub1(self, template_provided, escapeSpecial, legacyEscapeTreatment, expectedPattern):
+        template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
+        print(f"template [{template_provided}]")
+
+        print("max_placeholders", template1._max_placeholders )
+        print("max_rnds", template1._max_rnds_needed)
+        print("placeholders", template1._placeholders_needed )
+        print("bounds", template1._template_rnd_bounds)
+
+        print("templates", template1.templates)
+
+        TEST_ROWS = 100
+
+        arr = np.arange(TEST_ROWS)
+
+        template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
+
+        print("choices", template_choices)
+        print("rnd bounds", template_rnd_bounds)
+        print("template_rnds", template_rnds)
+
+        assert len(template_choices) == len(template_rnds)
+        assert len(template_choices) == len(template_rnd_bounds)
+
+        for ix in range(len(template_choices)):
+            bounds = template_rnd_bounds[ix]
+            rnds = template_rnds[ix]
+
+            assert len(bounds) == len(rnds)
+
+            for iy in range(len(bounds)):
+                assert bounds[iy] == -1 or (rnds[iy] < bounds[iy])
+
+
+        results = template1.pandasGenerateText(arr)
+        assert results is not None
+
+        results_list = results.tolist()
+
+        results_rows = len(results_list)
+        assert results_rows == TEST_ROWS
+
+        for r in range(len(results)):
+            result_str = results[r]
+            assert result_str is not None and isinstance(result_str, str)
+            assert len(result_str) >= 0
+
+        print("results")
+        for i in range(len(results)):
+            print(f"{i}: '{results[i]}'")
+
+    @pytest.mark.parametrize("template_provided, escapeSpecial, legacyEscapeTreatment,expectedPattern ",
+                             [ (r'\\w \\w|\\w a. \\w', False, False, ""),
+                                 (r'\\w \\w|\\w a. \\w', False, True, ""),
+                                 (r'\w \w|\w a. \w', False, False, ""),
+                                 (r'\w.\w@\w.com', False, False, ""),
+                                 (r'\n-\n', False, False, ""),
+                             ])
+    def test_value_sub2(self, template_provided, escapeSpecial, legacyEscapeTreatment, expectedPattern):
+        template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
+        print(f"template [{template_provided}]")
+
+        print("max_placeholders", template1._max_placeholders )
+        print("max_rnds", template1._max_rnds_needed)
+        print("placeholders", template1._placeholders_needed )
+        print("bounds", template1._template_rnd_bounds)
+
+        print("templates", template1.templates)
+
+        TEST_ROWS = 100
+
+        arr = np.arange(TEST_ROWS)
+
+        template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
+
+        print("choices", template_choices)
+        print("rnd bounds", template_rnd_bounds)
+        print("template_rnds", template_rnds)
+
+        assert len(template_choices) == len(template_rnds)
+        assert len(template_choices) == len(template_rnd_bounds)
+
+        for ix in range(len(template_choices)):
+            bounds = template_rnd_bounds[ix]
+            rnds = template_rnds[ix]
+
+            assert len(bounds) == len(rnds)
+
+            for iy in range(len(bounds)):
+                assert bounds[iy] == -1 or (rnds[iy] < bounds[iy])
+
+
+        results = template1.pandasGenerateText(arr)
+        assert results is not None
+
+        results_list = results.tolist()
+
+        results_rows = len(results_list)
+        assert results_rows == TEST_ROWS
+
+        for r in range(len(results)):
+            result_str = results[r]
+            assert result_str is not None and isinstance(result_str, str)
+            assert len(result_str) >= 0
+
+        print("results")
+        for i in range(len(results)):
+            print(f"{i}: '{results[i]}'")
+
+
     @pytest.mark.parametrize("template_provided, escapeSpecial, useTemplateObject",
                              [ (r'\w aAdDkK \w', False, False),
 

From 60743e9eca34b323d538e14bf212b22f1296b542 Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Mon, 20 Mar 2023 23:38:11 -0700
Subject: [PATCH 11/12] wip

---
 docs/source/textdata.rst     | 4 ++++
 tests/test_text_templates.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/source/textdata.rst b/docs/source/textdata.rst
index 9393da2b..b937f52c 100644
--- a/docs/source/textdata.rst
+++ b/docs/source/textdata.rst
@@ -168,5 +168,9 @@ when applied to the values zero to 999. This conforms to the preferred style goi
 will not be treated as a special char.
 
 .. note::
+          The legacy mode of operation has a bug where the template sequence r'\\a' produces the same result as r'\a'.
+          This can be disabled by setting the parameter `legacyEscapeTreatment` to False on the TemplateTextGenerator
+          object. It is true by default.
+
 
 
diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
index 40b2aecf..fa3e1362 100644
--- a/tests/test_text_templates.py
+++ b/tests/test_text_templates.py
@@ -274,7 +274,7 @@ def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
                                (r'\\a', True, False, r"\\[a-z]"),
                                (r'\\a', False, False, r"\\[a-z]"),
                                (r'\\a', False, True, r"[a-z]"),
-                               (r'\\a', True, True, r"\\[a-z]"),
+                               (r'\\a', True, True, r"[a-z]"),
                                ])
     def test_escape_treatment(self, templateProvided, escapeSpecial, legacyEscapeTreatment, expectedPattern):
 

From 9a4ed7b987d8bd525ac257bb39d3ceeb3e2e267a Mon Sep 17 00:00:00 2001
From: ronanstokes-db <ronan.stokes@databricks.com>
Date: Tue, 21 Mar 2023 00:13:56 -0700
Subject: [PATCH 12/12] wip

---
 dbldatagen/text_generators.py |  1 -
 tests/test_text_templates.py  | 28 ++++++++++------------------
 2 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/dbldatagen/text_generators.py b/dbldatagen/text_generators.py
index 7a56d80f..7a68024a 100644
--- a/dbldatagen/text_generators.py
+++ b/dbldatagen/text_generators.py
@@ -267,7 +267,6 @@ def __init__(self, template, escapeSpecialChars=False, extendedWordList=None, le
             assert (k is not None) and isinstance(k, str) and len(k) > 0, "key must be non-empty string"
             assert v is not None and isinstance(v, tuple) and len(v) == 2, "value must be tuple of length 2"
             mapping_length, mappings = v
-            print(type(mappings))
             assert isinstance(mapping_length, int), "mapping length must be of type int"
             assert isinstance(mappings, list) or isinstance(mappings,  np.ndarray),\
                 "mappings are lists or numpy arrays"
diff --git a/tests/test_text_templates.py b/tests/test_text_templates.py
index fa3e1362..8396af83 100644
--- a/tests/test_text_templates.py
+++ b/tests/test_text_templates.py
@@ -212,14 +212,6 @@ def test_rnd_compute(self, template_provided, escapeSpecial, useTemplateObject):
                              ])
     def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
         template1 = TemplateGenerator(template_provided, escapeSpecialChars=escapeSpecial)
-        print(f"template [{template_provided}]")
-
-        print("max_placeholders", template1._max_placeholders )
-        print("max_rnds", template1._max_rnds_needed)
-        print("placeholders", template1._placeholders_needed )
-        print("bounds", template1._template_rnd_bounds)
-
-        print("templates", template1.templates)
 
         TEST_ROWS = 100
 
@@ -227,10 +219,6 @@ def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
 
         template_choices, template_rnd_bounds, template_rnds = template1._prepare_random_bounds(arr)
 
-        print("choices", template_choices)
-        print("rnd bounds", template_rnd_bounds)
-        print("template_rnds", template_rnds)
-
         assert len(template_choices) == len(template_rnds)
         assert len(template_choices) == len(template_rnd_bounds)
 
@@ -263,17 +251,21 @@ def test_use_pandas(self, template_provided, escapeSpecial, useTemplateObject):
 
     @pytest.mark.parametrize("templateProvided, escapeSpecial, legacyEscapeTreatment,expectedPattern ",
                              [ (r'\\w \w', False, True, r"[a-z]+ [a-z]+"),
-                               (r'\\w \w', False, False, r"\\[a-z]+ [a-z]+"),
+                               (r'\\w \w', False, False, r"\\w [a-z]+"),
+                               (r'\\\w \w', False, False, r"\\[a-z]+ [a-z]+"),
                                (r'\\w \w', True, True, r"[a-z]+ [a-z]+"),
-                               (r'\\w \w', True, False, r"\\[a-z]+ [a-z]+"),
+                               (r'\\w \w', True, False, r"\\w [a-z]+"),
+                               (r'\\\w \w', True, False, r"\\[a-z]+ [a-z]+"),
                                (r'\n-\n', False, False, r"[0-9]+-[0-9]+"),
                                (r'\\n-\n', False, True, r"[0-9]+-[0-9]+"),
-                               (r'\\n-\n', False, False, r"\\[0-9]+-[0-9]+"),
+                               (r'\\n-\n', False, False, r"\\n-[0-9]+"),
+                               (r'\\\n-\n', False, False, r"\\[0-9]+-[0-9]+"),
                                (r'\\n-\n', True, True, r"[0-9]+-[0-9]+"),
-                               (r'\\n-\n', True, False, r"\\[0-9]+-[0-9]+"),
-                               (r'\\a', True, False, r"\\[a-z]"),
+                               (r'\\n-\n', True, False, r"\\n-[0-9]+"),
+                               (r'\\\n-\n', True, False, r"\\[0-9]+-[0-9]+"),
+                               (r'\\\a', True, False, r"\\[a-z]"),
                                (r'\\a', False, False, r"\\[a-z]"),
-                               (r'\\a', False, True, r"[a-z]"),
+                               (r'\\a c', False, True, r"[a-z] c"),
                                (r'\\a', True, True, r"[a-z]"),
                                ])
     def test_escape_treatment(self, templateProvided, escapeSpecial, legacyEscapeTreatment, expectedPattern):