nomulus/config/presubmits.py

# Copyright 2019 The Nomulus Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Google-style presubmits for the Nomulus project.

These aren't built in to the static code analysis tools we use (e.g. Checkstyle,
Error Prone) so we must write them manually.
"""

import json
import os
from typing import List, Tuple
import sys
import textwrap
import re

# We should never analyze any generated files
UNIVERSALLY_SKIPPED_PATTERNS = {"/build/", "cloudbuild-caches", "/out/", ".git/",
     ".gradle/", "/dist/", "/console-alpha/", "/console-crash/", "/console-qa",
     "/console-sandbox", "/console-production", "karma.conf.js", "polyfills.ts",
     "test.ts", "/docs/console-endpoints/", "/bin/generated-sources/",
     "/bin/generated-test-sources/", "src/main/generated", "src/test/generated"}
# We can't rely on CI to have the Enum package installed so we do this instead.
FORBIDDEN = 1
REQUIRED = 2

# The list of expected json packages and their licenses.
# These should be one of the allowed licenses in:
# config/dependency-license/allowed_licenses.json
EXPECTED_JS_PACKAGES = [
    'google-closure-library',   # Owned by Google, Apache 2.0
]


class PresubmitCheck:

  def __init__(self,
               regex,
               included_extensions,
               skipped_patterns,
               regex_type=FORBIDDEN):
    """Define a presubmit check for a particular set of files,

        The provided prefix should always or never be included in the files.

        Args:
            regex: the regular expression to forbid or require
            included_extensions: a tuple of extensions that define which files
              we run over
            skipped_patterns: a set of patterns that will cause any files that
              include them to be skipped
            regex_type: either FORBIDDEN or REQUIRED--whether or not the regex
              must be present or cannot be present.
    """
    self.regex = regex
    self.included_extensions = included_extensions
    self.skipped_patterns = UNIVERSALLY_SKIPPED_PATTERNS.union(skipped_patterns)
    self.regex_type = regex_type

  def fails(self, file):
    """ Determine whether or not this file fails the regex check.

    Args:
        file: the full path of the file to check
    """
    if not file.endswith(self.included_extensions):
      return False
    for pattern in self.skipped_patterns:
      if pattern in file:
        return False
    with open(file, "r", encoding='utf8') as f:
      file_content = f.read()
      matches = re.match(self.regex, file_content, re.DOTALL)
      if self.regex_type == FORBIDDEN:
        return matches
      return not matches


PRESUBMITS = {
    # License check
    PresubmitCheck(
        r".*Copyright 20\d{2} The Nomulus Authors\. All Rights Reserved\.",
        ("java", "js", "sql", "py", "sh", "gradle", "ts", "ftl"), {
            ".git", "/build/", "node_modules/", "LoggerConfig.java", "registrar_bin.",
            "registrar_dbg.", "google-java-format-diff.py",
            "nomulus.golden.sql", "javascript/checks.js"
        }, REQUIRED):
        "File did not include the license header.",

    # Files must end in a newline
    PresubmitCheck(r".*\n$", ("java", "js", "sql", "py", "sh", "gradle", "ts", "xml", "ftl"),
                   {"node_modules/", ".idea"}, REQUIRED):
        "Source files must end in a newline.",

    # Files must not end with extraneous blank lines
    PresubmitCheck(
        r".*\n\n$",
        ("java", "js", "soy", "sql", "py", "sh", "gradle", "ts", "xml"),
        {"node_modules/", ".idea", "nomulus.golden.sql"},
    ):
        "Source files must not end with extraneous blank lines.",

    # Duplicate empty lines
    PresubmitCheck(
        r".*\n\n\n.*",
        ("java", "js", "soy", "sh", "gradle", "ts", "xml"),
        {"node_modules/", ".idea"},
    ):
        "Source files must not contain duplicate empty lines.",

    # System.(out|err).println should only appear in tools/ or load-testing/
    PresubmitCheck(
        r".*\bSystem\s*\.\s*(?:out|err)\s*\.\s*print.*", "java", {
            "/tools/", "/example/", "/load-testing/",
            "RegistryTestServerMain.java", "TestServerExtension.java"
        }):
        "System.(out|err).println is only allowed in tools/ packages. Please "
        "use a logger instead.",

    PresubmitCheck(
        r".*\nimport\s+(?:static\s+)?.*\.shaded\..*",
        "java",
        {"/node_modules/"},
    ):
        "Do not use shaded dependencies",
    PresubmitCheck(
        r".*com\.google\.common\.truth\.Truth8.*",
        "java",
        {"/node_modules/"},
    ):
        "Truth8 is deprecated. Use Truth instead.",
    PresubmitCheck(
        r".*java\.util\.Date.*",
        "java",
        {
            "/node_modules/",
            "JpaTransactionManagerImpl.java",
            "DateTimeUtils.java",
            "SelfSignedCaCertificate.java",
            "X509Utils.java",
            "TmchCertificateAuthority.java",
            "DelegatedCredentials.java",
            "SslInitializerTestUtils.java"
        },
    ):
        "Do not use java.util.Date. Use classes in java.time package instead.",
    PresubmitCheck(
        r".*com\.google\.api\.client\.http\.HttpStatusCodes.*",
        "java",
        {"/node_modules/"},
    ):
        "Use status code from jakarta.servlet.http.HttpServletResponse.",
    PresubmitCheck(
        r".*mock\(\s*Response\.class\s*\).*",
        "java",
        {"/node_modules/"},
    ):
        "Do not mock Response, use FakeResponse.",
    PresubmitCheck(
        r".*javax\.servlet\..*",
        "java",
        {"/node_modules/"},
    ):
        "Do not use javax.servlet.* Use jakarta.servlet.* instead.",
    PresubmitCheck(
        r".*javax\.inject\..*",
        "java",
        {"/node_modules/"},
    ):
        "Do not use javax.inject.* Use jakarta.inject.* instead.",
    PresubmitCheck(
        r".*import\s+jakarta\.persistence\.(?:Pre|Post)(?:Persist|Load|Remove|Update)\s*;",
        "java",
        {"EntityCallbacksListener.java"},
    ):
        "Hibernate lifecycle events aren't called for embedded entities, so it's "
        "usually best to avoid them. Instead, use the annotations defined in "
        "EntityCallbacksListener.java",
    PresubmitCheck(
        r".*\.isEqualTo\(\s*Optional\.of\(.*",
        "java",
        {},
    ):
        "Do not use .isEqualTo(Optional.of(...)). Use Truth's .hasValue(...) instead.",
    PresubmitCheck(
        r".*java\.time\.ZonedDateTime.*",
        "java",
        {},
    ):
        "Do not use java.time.ZonedDateTime. Use java.time.OffsetDateTime per go/avoid-zdt.",
    PresubmitCheck(
        r".*ZoneId\.of\(\s*\"UTC\"\s*\).*",
        "java",
        {},
    ):
        "Do not use ZoneId.of(\"UTC\"). Use java.time.ZoneOffset.UTC.",
    PresubmitCheck(
        r".*org\.joda\.time.*",
        "java",
        {"SafeBrowsingTransforms.java"},
    ):
        "Do not use Joda-Time. Use java.time instead.",
}

# Note that this regex only works for one kind of Flyway file.  If we want to
# start using "R" and "U" files we'll need to update this script.
FLYWAY_FILE_RX = re.compile(r'V(\d+)__.*')


def get_seqnum(filename: str, location: str) -> int:
  """Extracts the sequence number from a filename."""
  m = FLYWAY_FILE_RX.match(filename)
  if m is None:
    raise ValueError('Illegal Flyway filename: %s in %s' % (filename, location))
  return int(m.group(1))


def files_by_seqnum(files: List[str], location: str) -> List[Tuple[int, str]]:
  """Returns the list of seqnum, filename sorted by sequence number."""
  return [(get_seqnum(filename, location), filename) for filename in files]


def has_valid_order(indexed_files: List[Tuple[int, str]], location: str) -> bool:
  """Verify that sequence numbers are in order without gaps or duplicates.

  Args:
    files: List of seqnum, filename for a list of Flyway files.
    location: Where the list of files came from (for error reporting).

  Returns:
    True if the file list is valid.
  """
  last_index = 0
  valid = True
  for seqnum, filename in indexed_files:
    if seqnum == last_index:
      print('duplicate Flyway file sequence number found in %s: %s' %
            (location, filename))
      valid = False
    elif seqnum < last_index:
      print('File %s in %s is out of order.' % (filename, location))
      valid = False
    elif seqnum != last_index + 1:
      print('Missing Flyway sequence number %d in %s.  Next file is %s' %
            (last_index + 1, location, filename))
      valid = False
    last_index = seqnum
  return valid


def verify_flyway_index():
  """Verifies that the Flyway index file is in sync with the directory."""
  success = True

  # Sort the files in the Flyway directory by their sequence number.
  files = sorted(
      files_by_seqnum(os.listdir('db/src/main/resources/sql/flyway'),
                      'Flyway directory'))

  # Make sure that there are no gaps and no duplicate sequence numbers in the
  # files themselves.
  if not has_valid_order(files, 'Flyway directory'):
    success = False

  # Remove the sequence numbers and compare against the index file contents.
  files = [filename[1] for filename in sorted(files)]
  with open('db/src/main/resources/sql/flyway.txt', encoding='utf8') as index:
    indexed_files = index.read().splitlines()
  if files != indexed_files:
    unindexed = set(files) - set(indexed_files)
    if unindexed:
      print('The following Flyway files are not in flyway.txt: %s' % unindexed)

    nonexistent = set(indexed_files) - set(files)
    if nonexistent:
      print('The following files are in flyway.txt but not in the Flyway '
            'directory: %s' % nonexistent)

    # Do an ordering check on the index file (ignore the result, we're failing
    # anyway).
    has_valid_order(files_by_seqnum(indexed_files, 'flyway.txt'), 'flyway.txt')
    success = False

  if not success:
    print('Please fix any conflicts and run "./nom_build :db:generateFlywayIndex"')

  return not success


def get_files():
  for root, dirnames, filenames in os.walk("."):
    for filename in filenames:
      yield os.path.join(root, filename)


if __name__ == "__main__":
  failed = False
  for file in get_files():
    error_messages = []
    for presubmit, error_message in PRESUBMITS.items():
      if presubmit.fails(file):
        error_messages.append(error_message)

    if error_messages:
      failed = True
      print("%s had errors: \n  %s" % (file, "\n  ".join(error_messages)))

  # And now for something completely different: check to see if the Flyway
  # index is up-to-date.  It's quicker to do it here than in the unit tests:
  # when we put it here it fails fast before all of the tests are run.
  failed |= verify_flyway_index()

  if failed:
    sys.exit(1)