update README and add metadata updater to samples repo (#304)

tools/MetadataUpdater/README.md

@@ -0,0 +1,53 @@
+
+## Metadata updater script
+
+This script updates the `README.metadata.json` files for any samples that it is given.
+
+### How to use this script
+
+Navigate to the top-level directory of this repository (`arcgis-maps-sdk-kotlin-samples`).
+
+The script has two types of arguments:
+* `-m` or `--multiple` to recreate metadata files for all samples in a given directory.
+```
+# recreates all metadata files for samples
+python3 tools/MetdataUpdater/metadata_updater.py -m ../arcgis-maps-sdk-kotlin-samples/samples
+```
+* `-s` or `--single` to recreate a metadata file for a single given sample. The argument should provide the language directory name and the sample directory name.
+```
+# recreates the metadata file for the kotlin sample "Add features feature service"
+python3 tools/MetadataUpdater/metadata_updater.py -s ../arcgis-maps-sdk-kotlin-samples/samples/add-features-feature-service
+```
+
+**Note:** The script cannot create a metadata file from scratch. You should first create a file in the sample's directory called `README.metadata.json`. The contents of the file can be
+```
+{
+}
+```
+
+When recreating single metadata files, if any of the following entries are not present or empty, they will be created and given the value "TODO". This is because they cannot be filled in by the script. Please remove the "TODO" and update with the correct info or remove the entry altogether before merging.
+* category
+* provision_from
+* provision_to
+* redirect_from
+
+### How it works
+
+To update all sample metadata files in a directory:
+
+1. Loop through the subfolders of the provided directory
+2. A `MetadataUpdater` is created, passing in the subfolder's path, with class fields for each key of the output json.
+3. Populate fields from the existing `README.metadata.json`:
+  * Check for a `category` key and write it to the updater's `self.category` field.
+  * For each of `provision_from`, `provision_to`, and `redirect_from`, check if the key exists, and if it does, write it to the corresponding field of the updater.
+4. Populate fields from the sample's `README.md`:
+  * Split the readme by two hash symbols `##` to find the headings of each section.
+  * Get the title and description by parsing the head (first section) of the readme.
+  * Create the `formal_name` property by converting the title to Pascal case.
+  * Parse the APIs and tags by cleaning up white space and separators in the readme.
+5. Populate fields from the sample's file paths:
+  * To get the screenshot, traverse the immediate files inside the sample directory, looking for a file with the `.png` extension.
+  * To get the language and snippets, search recursively through the directory for files with the extension `.java` or `.kt`, ignoring the `/build/` directory.
+6. Create a dictionary. For each of the required metadata keys, create a key with a string title and a corresponding class field as its value.
+  * For the `category`, `provision_from`, `provision_to`, and `redirect_from` keys, check if that they are not empty in the updater's fields before adding them to the dictionary. If they are empty, set them to "TODO"
+7. Dump the dictionary to a json file.

tools/MetadataUpdater/metadata_updater.py

@@ -0,0 +1,311 @@
+import os
+import re
+import json
+import typing
+import argparse
+
+
+def check_special_char(string: str) -> bool:
+    """
+    Check if a string contains special characters.
+
+    :param string: The input string.
+    :return: True if there are special characters.
+    """
+    # regex = re.compile('[@_!#$%^&*()<>?/\\|}{~:]')
+    regex = re.compile('[@_!#$%^&*<>?|/\\}{~:]')
+    if not regex.search(string):
+        return False
+    return True
+
+
+def parse_head(head_string: str) -> (str, str):
+    """
+    Parse the `Title` section of README file and get the title and description.
+
+    :param head_string: A string containing title, description and images.
+    :return: Stripped title and description strings.
+    """
+    parts = list(filter(bool, head_string.splitlines()))
+    if len(parts) < 3:
+        raise Exception('README description parse failure!')
+    title = parts[0].lstrip('# ').rstrip()
+    description = parts[1].strip()
+    return title, description
+
+
+def parse_apis(apis_string: str) -> typing.List[str]:
+    """
+    Parse the `Relevant API` section and get a list of APIs.
+
+    :param apis_string: A string containing all APIs.
+    :return: A sorted list of stripped API names.
+    """
+    apis = list(filter(bool, apis_string.splitlines()))
+    if not apis:
+        raise Exception('README Relevant API parse failure!')
+    return sorted([api.lstrip('*- ').rstrip() for api in apis])
+
+
+def parse_tags(tags_string: str) -> typing.List[str]:
+    """
+    Parse the `Tags` section and get a list of tags.
+
+    :param tags_string: A string containing all tags, with comma or newline as delimiter.
+    :return: A sorted list of stripped tags.
+    """
+    tags = re.split(r'[,\n]', tags_string)
+    if not tags:
+        raise Exception('README Tags parse failure!')
+    tags = [x for x in tags if x != '']
+    return sorted([tag.strip() for tag in tags])
+
+
+def get_folder_name_from_path(path: str) -> str:
+    """
+    Get the folder name from a full path.
+
+    :param path: A string of a full/absolute path to a folder.
+    :return: The folder name.
+    """
+    return os.path.normpath(path).split(os.path.sep)[-1]
+
+
+class MetadataUpdater:
+
+    def __init__(self, folder_path: str, single_update: bool = False):
+        """
+        The standard format of metadata.json for Android platform. Read more at:
+        https://devtopia.esri.com/runtime/common-samples/wiki/README.metadata.json
+        """
+        self.category = ''          # Populate from json.
+        self.description = ''       # Populate from README.
+        self.formal_name = ''       # Populate from README.
+        self.ignore = False         # Default to False.
+        self.images = []            # Populate from folder paths.
+        self.keywords = []          # Populate from README.
+        self.language = ''          # Populate from folder paths.
+        self.redirect_from = []     # Populate from json.
+        self.relevant_apis = []     # Populate from README.
+        self.snippets = []          # Populate from folder paths.
+        self.title = ''             # Populate from README.
+
+        self.folder_path = folder_path
+        self.folder_name = get_folder_name_from_path(folder_path)
+        self.readme_path = os.path.join(folder_path, 'README.md')
+        self.json_path = os.path.join(folder_path, 'README.metadata.json')
+
+        self.single_update = single_update
+
+    def get_source_code_paths(self) -> typing.List[str]:
+        """
+        Traverse the directory and get all filenames for source code.
+        Ignores any code files in the `/build/` directory.
+
+        :return: A list of java or kotlin source code filenames starting from `/src/`.
+        """
+        results = []
+        for dp, dn, filenames in os.walk(self.folder_path):
+            if ("/build/" not in dp):
+                for file in filenames:
+                    extension = os.path.splitext(file)[1]
+                    if extension in ['.java'] or extension in ['.kt']:
+                        # get the programming language of the sample
+                        self.language = 'java' if extension in ['.java'] else 'kotlin'
+                        # get the snippet path
+                        snippet = os.path.join(dp, file)
+                        if snippet.startswith(self.folder_path):
+                            # add 1 to remove the leading slash
+                            snippet = snippet[len(self.folder_path)+1:]
+                        if "ViewModel" in snippet:
+                            results.insert(0, snippet)
+                        else:
+                            results.append(snippet)
+        if not results:
+            raise Exception('Unable to get java/kotlin source code paths.')
+        return results
+
+    def get_images_paths(self):
+        """
+        Traverse the directory and get all filenames for images in the top level directory.
+
+        :return: A list of image filenames.
+        """
+        results = []
+        list_subfolders_with_paths = [f.name for f in os.scandir(self.folder_path) if f.is_file()]
+        for file in list_subfolders_with_paths:
+            if os.path.splitext(file)[1].lower() in ['.png']:
+                results.append(file)
+        if not results:
+            raise Exception('Unable to get images paths.')
+        return sorted(results)
+
+    def populate_from_json(self) -> None:
+        """
+        Read 'category' and 'redirect_from'
+        fields from json, as they should not be changed.
+        """
+        try:
+            json_file = open(self.json_path, 'r')
+            json_data = json.load(json_file)
+        except Exception as err:
+            print(f'Error reading JSON - {self.json_path} - {err}')
+            raise err
+        else:
+            json_file.close()
+
+        keys = json_data.keys()
+        for key in ['category']:
+            if key in keys:
+                setattr(self, key, json_data[key])
+        if 'redirect_from' in keys:
+            if isinstance(json_data['redirect_from'], str):
+                self.redirect_from = [json_data['redirect_from']]
+            elif isinstance(json_data['redirect_from'], typing.List):
+                self.redirect_from = json_data['redirect_from']
+            else:
+                print(f'No redirect_from in - {self.json_path}, abort.')
+
+    def populate_from_readme(self) -> None:
+        """
+        Read and parse the sections from README, and fill in the 'title',
+        'description', 'relevant_apis' and 'keywords' fields in the dictionary
+        for output json.
+        """
+        try:
+            readme_file = open(self.readme_path, 'r')
+            # read the readme content into a string
+            readme_contents = readme_file.read()
+        except Exception as err:
+            print(f"Error reading README - {self.readme_path} - {err}.")
+            raise err
+        else:
+            readme_file.close()
+
+        # Use regex to split the README by exactly 2 pound marks, so that they
+        # are separated into paragraphs.
+        pattern = re.compile(r'^#{2}(?!#)\s(.*)', re.MULTILINE)
+        readme_parts = re.split(pattern, readme_contents)
+        try:
+            api_section_index = readme_parts.index('Relevant API') + 1
+            tags_section_index = readme_parts.index('Tags') + 1
+            self.title, self.description = parse_head(readme_parts[0])
+            # create a formal name key from a pascal case version of the title
+            # with parentheses removed.
+            formal_name = ''.join(x for x in self.title.title() if not x.isspace())
+            self.formal_name = re.sub('[()]','', formal_name)
+
+            if check_special_char(self.title + self.description):
+                print(f'Info: special char in README - {self.folder_name}')
+            self.relevant_apis = parse_apis(readme_parts[api_section_index])
+            keywords = parse_tags(readme_parts[tags_section_index])
+            # Do not include relevant apis in the keywords
+            self.keywords = [w for w in keywords if w not in self.relevant_apis]
+
+            # This is left in from the iOS script:
+            # "It combines the Tags and the Relevant APIs in the README."
+            # See /runtime/common-samples/wiki/README.metadata.json#keywords
+            self.keywords += self.relevant_apis
+        except Exception as err:
+            print(f'Error parsing README - {self.readme_path} - {err}.')
+            raise err
+
+    def populate_from_paths(self) -> None:
+        """
+        Populate source code and image filenames from a sample's folder.
+        """
+        try:
+            self.images = self.get_images_paths()
+            self.snippets = self.get_source_code_paths()
+        except Exception as err:
+            print(f"Error parsing paths - {self.folder_name} - {err}.")
+            raise err
+
+    def flush_to_json(self, path_to_json: str) -> None:
+        """
+        Write the metadata to a json file.
+
+        :param path_to_json: The path to the json file.
+        """
+        data = dict()
+
+        if not self.category and self.single_update:
+            data["category"] = "TODO"
+        else:
+            data["category"] = self.category
+
+        data["description"] = self.description
+        data["formal_name"] = self.formal_name
+        data["ignore"] = self.ignore
+        data["images"] = self.images
+        data["keywords"] = self.keywords
+        data["language"] = self.language
+
+        if self.redirect_from and self.redirect_from[0] is not '':
+            data["redirect_from"] = self.redirect_from
+        elif self.single_update:
+            data["redirect_from"] = "TODO"
+
+        data["relevant_apis"] = self.relevant_apis
+        data["snippets"] = self.snippets
+        data["title"] = self.title
+
+        with open(path_to_json, 'w+') as json_file:
+            json.dump(data, json_file, indent=4, sort_keys=True)
+            json_file.write('\n')
+
+
+def update_1_sample(path: str):
+    """
+    Fixes 1 sample's metadata by running the script on a single sample's directory.
+    """
+    single_updater = MetadataUpdater(path, True)
+    try:
+        single_updater.populate_from_json()
+        single_updater.populate_from_readme()
+        single_updater.populate_from_paths()
+    except Exception:
+        print(f'Error populate failed for - {single_updater.folder_name}.')
+        return
+    single_updater.flush_to_json(os.path.join(path, 'README.metadata.json'))
+
+
+def main():
+    # Initialize parser.
+    msg = 'Metadata helper script. Run it against the top level folder of an ' \
+          'Android platform language (ie. kotlin or java) with the -m flag ' \
+          'or against a single sample using the -s flag and passing in eg. kotlin/my-sample-dir'
+    parser = argparse.ArgumentParser(description=msg)
+    parser.add_argument('-m', '--multiple', help='input directory of the language')
+    parser.add_argument('-s', '--single', help='input directory of the sample')
+    args = parser.parse_args()
+
+    if args.multiple:
+        category_root_dir = args.multiple
+        category_name = get_folder_name_from_path(category_root_dir)
+        print(f'Processing category - `{category_name}`...')
+
+        list_subfolders_with_paths = [f.path for f in os.scandir(category_root_dir) if f.is_dir()]
+        for current_path in list_subfolders_with_paths:
+            print(current_path)
+            updater = MetadataUpdater(current_path)
+            try:
+                updater.populate_from_json()
+                updater.populate_from_readme()
+                updater.populate_from_paths()
+            except Exception:
+                print(f'Error populate failed for - {updater.folder_name}.')
+                continue
+            updater.flush_to_json(updater.json_path)
+    elif args.single:
+        update_1_sample(args.single)
+    else:
+        update_1_sample()
+        print('Invalid arguments, abort.')
+
+
+if __name__ == '__main__':
+    # Use main function for a full category.
+    main()
+    # Use test function for a single sample.
+    # update_1_sample()