Support TOML test files (#2370)

* refactor: remove many JSON.parse()

The old code use multiple times JSON.parse()

The code is now more clear that there are:
callbackParameter.rawFile
callbackParameter.jsonObj

* update the axios to get the 'arraybuffer' as download.

The old code get the default JSON object as response.

* add toml for schema test validation

* hide count for fullstrict mode if only one AJV schema is scan

The output is not correct any more

* add one toml test file

* update package.json

* update pull request template

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,7 +5,7 @@ Before continuing, please read the guidelines:
 https://github.com/SchemaStore/schemastore/blob/master/CONTRIBUTING.md
 
 Adding a JSON schema file to the catalog is required.
-It is recommended to add tests.
+Add tests files. (.json, .yml, .yaml or .toml)
 Use the lowest possible schema draft needed, preferably Draft v4.
 JSON formatted according to the .editorconfig settings.
 

src/Gruntfile.cjs

@@ -6,6 +6,7 @@ const AjvDraft06And07 = require('ajv')
 const Ajv2019 = require('ajv/dist/2019')
 const Ajv2020 = require('ajv/dist/2020')
 const tv4 = require('tv4')
+const TOML = require('@ltd/j-toml')
 const YAML = require('yaml')
 const pt = require('path')
 const fs = require('fs')
@@ -67,6 +68,7 @@ module.exports = function (grunt) {
   async function remoteSchemaFile (schemaOnlyScan, showLog = true) {
     const axios = require('axios').default
     const schemas = catalog.schemas
+    const responseType = 'arraybuffer'
 
     for (const { url } of schemas) {
       if (url.startsWith(urlSchemaStore)) {
@@ -74,12 +76,13 @@ module.exports = function (grunt) {
         continue
       }
       try {
-        const response = await axios(url)
+        const response = await axios.get(url, { responseType })
         if (response.status === 200) {
           const parsed = new URL(url)
           const callbackParameter = {
             jsonName: pt.basename(parsed.pathname),
-            rawFile: JSON.stringify(response.data),
+            jsonObj: JSON.parse(response.data.toString()),
+            rawFile: response.data,
             urlOrFilePath: url,
             schemaScan: true
           }
@@ -163,9 +166,17 @@ module.exports = function (grunt) {
         // Some schema files must be ignored.
         if (canThisTestBeRun(schemaFileName) &&
             !skipThisFileName(schemaFileName)) {
+          const buffer = skipReadFile ? undefined : fs.readFileSync(schemaFullPathName)
+          let jsonObj_
+          try {
+            jsonObj_ = buffer ? JSON.parse(buffer.toString()) : undefined
+          } catch (err) {
+            throwWithErrorText([`JSON file ${schemaFullPathName} did not parse correctly.`, err])
+          }
           const callbackParameter = {
             // Return the real Raw file for BOM file test rejection
-            rawFile: skipReadFile ? undefined : fs.readFileSync(schemaFullPathName),
+            rawFile: buffer,
+            jsonObj: jsonObj_,
             jsonName: pt.basename(schemaFullPathName),
             urlOrFilePath: schemaFullPathName,
             schemaScan: onlySchemaScan
@@ -177,16 +188,36 @@ module.exports = function (grunt) {
 
     // Scan one test folder for all the files inside it
     const scanOneTestFolder = (schemaName, testDir, testPassScan, testPassScanDone) => {
-      const loadTestFile = (testFileNameWithPath) => {
+      const loadTestFile = (testFileNameWithPath, buffer) => {
         // Test files have extension '.json' or else it must be a YAML file
-        if (testFileNameWithPath.endsWith('.json')) {
-          return grunt.file.read(testFileNameWithPath)
-        } else { // YAML file
-          try {
-            return JSON.stringify(YAML.parse(fs.readFileSync(testFileNameWithPath, 'utf8')))
-          } catch (e) {
-            throwWithErrorText([`Can't read/decode yaml file: ${testFileNameWithPath}`, e])
-          }
+        const fileExtension = testFileNameWithPath.split('.').pop()
+        switch (fileExtension) {
+          case 'json':
+            try {
+              return JSON.parse(buffer.toString())
+            } catch (err) {
+              throwWithErrorText([`JSON file ${testFileNameWithPath} did not parse correctly.`, err])
+            }
+            break
+          case 'yaml':
+          case 'yml':
+            try {
+              return YAML.parse(buffer.toString())
+            } catch (e) {
+              throwWithErrorText([`Can't read/decode yaml file: ${testFileNameWithPath}`, e])
+            }
+            break
+          case 'toml':
+            try {
+              // { bigint: false } or else toml variable like 'a = 3' will return as 'a = 3n'
+              // This creates an error because the schema expect an integer 3 and not 3n
+              return TOML.parse(buffer.toString(), { bigint: false })
+            } catch (e) {
+              throwWithErrorText([`Can't read/decode toml file: ${testFileNameWithPath}`, e])
+            }
+            break
+          default:
+            throwWithErrorText([`Unknown file extension: ${fileExtension}`])
         }
       }
 
@@ -218,8 +249,10 @@ module.exports = function (grunt) {
           throwWithErrorText([`Found non test file inside test folder: ${testFileFullPathName}`])
         }
         if (!skipThisFileName(pt.basename(testFileFullPathName))) {
+          const buffer = skipReadFile ? undefined : fs.readFileSync(testFileFullPathName)
           const callbackParameter = {
-            rawFile: skipReadFile ? undefined : loadTestFile(testFileFullPathName),
+            rawFile: buffer,
+            jsonObj: skipReadFile ? undefined : loadTestFile(testFileFullPathName, buffer),
             jsonName: pt.basename(testFileFullPathName),
             urlOrFilePath: testFileFullPathName,
             // This is a test folder scan process, not schema scan process
@@ -289,7 +322,7 @@ module.exports = function (grunt) {
       let schemaVersionStr = 'unknown'
       try {
         // select the correct AJV object for this schema
-        schemaToBeValidated = JSON.parse(callbackParameter.rawFile)
+        schemaToBeValidated = callbackParameter.jsonObj
         versionObj = schemaVersion.getObj(schemaToBeValidated)
 
         // What schema draft version is it?
@@ -321,7 +354,7 @@ module.exports = function (grunt) {
     }
 
     const processPositiveTestFile = (callbackParameter) => {
-      const testFile = JSON.parse(callbackParameter.rawFile)
+      const testFile = callbackParameter.jsonObj
       const validated = tv4.validate(testFile, schemaToBeValidated)
       if (tv4.missing.length > 0) {
         throwWithErrorText([`${textPositiveFailedTest}${callbackParameter.urlOrFilePath}`,
@@ -497,7 +530,7 @@ module.exports = function (grunt) {
       const fullStrictModeStr = fullStrictMode ? '(FullStrictMode)' : '(NotStrictMode)'
       try {
         // select the correct AJV object for this schema
-        schemaJson = JSON.parse(callbackParameter.rawFile)
+        schemaJson = callbackParameter.jsonObj
         versionObj = schemaVersion.getObj(schemaJson)
 
         // Get the correct AJV version
@@ -531,13 +564,7 @@ module.exports = function (grunt) {
     }
 
     const processTestFile = (callbackParameter, success, failure) => {
-      let json
-      try {
-        json = JSON.parse(callbackParameter.rawFile)
-      } catch (e) {
-        throwWithErrorText([`Error in parse test: ${callbackParameter.urlOrFilePath}`, e])
-      }
-      validate(json) ? success() : failure()
+      validate(callbackParameter.jsonObj) ? success() : failure()
     }
 
     const processPositiveTestFile = (callbackParameter) => {
@@ -664,7 +691,7 @@ module.exports = function (grunt) {
       countScan++
       let result
       try {
-        result = findDuplicatedPropertyKeys(callbackParameter.rawFile)
+        result = findDuplicatedPropertyKeys(JSON.stringify(callbackParameter.jsonObj))
       } catch (e) {
         throwWithErrorText([`Test file: ${callbackParameter.urlOrFilePath}`, e])
       }
@@ -773,7 +800,7 @@ module.exports = function (grunt) {
 
   grunt.registerTask('local_check_filename_extension', 'Dynamically check local schema/test file for filename extension', function () {
     const schemaFileExtension = ['.json']
-    const testFileExtension = ['.json', '.yml', '.yaml']
+    const testFileExtension = ['.json', '.yml', '.yaml', '.toml']
     let countScan = 0
     const x = (data, fileExtensionList) => {
       countScan++
@@ -868,7 +895,7 @@ module.exports = function (grunt) {
     const testLowerSchemaVersion = (callbackParameter) => {
       countScan++
       let versionIndexOriginal = 0
-      const schemaJson = JSON.parse(callbackParameter.rawFile)
+      const schemaJson = callbackParameter.jsonObj
 
       if (!('$schema' in schemaJson)) {
         // There is no $schema present in the file.
@@ -946,7 +973,7 @@ module.exports = function (grunt) {
       process_data: (callbackParameter) => {
         let obj
         try {
-          obj = getObj_(JSON.parse(callbackParameter.rawFile))
+          obj = getObj_(callbackParameter.jsonObj)
         } catch (e) {
           // suppress possible JSON.parse exception. It will be processed as obj = undefined
         }
@@ -992,13 +1019,7 @@ module.exports = function (grunt) {
     localSchemaFileAndTestFile({
       schemaOnlyScan (callbackParameter) {
         countScan++
-        let schemaJson
-        try {
-          schemaJson = JSON.parse(callbackParameter.rawFile)
-        } catch (err) {
-          throwWithErrorText([`Schema file ${callbackParameter.jsonName} did not parse correctly.`, err])
-        }
-        if (!('$schema' in schemaJson)) {
+        if (!('$schema' in callbackParameter.jsonObj)) {
           throwWithErrorText([`Schema file is missing '$schema' keyword => ${callbackParameter.jsonName}`])
         }
       }
@@ -1079,10 +1100,13 @@ module.exports = function (grunt) {
         countSchemaScanViaAJV++
       }
     })
-    const countFullStrictSchema = countSchemaScanViaAJV - schemaValidation.ajvNotStrictMode.length
-    const percent = (countFullStrictSchema / countSchemaScanViaAJV) * 100
-    grunt.log.ok('Schema in full strict mode to prevent any unexpected behaviours or silently ignored mistakes in user schemas.')
-    grunt.log.ok(`${countFullStrictSchema} of ${countSchemaScanViaAJV} (${Math.round(percent)}%)`)
+    // If only ONE AJV schema test is run then this calculation does not work.
+    if (countSchemaScanViaAJV !== 1) {
+      const countFullStrictSchema = countSchemaScanViaAJV - schemaValidation.ajvNotStrictMode.length
+      const percent = (countFullStrictSchema / countSchemaScanViaAJV) * 100
+      grunt.log.ok('Schema in full strict mode to prevent any unexpected behaviours or silently ignored mistakes in user schemas.')
+      grunt.log.ok(`${countFullStrictSchema} of ${countSchemaScanViaAJV} (${Math.round(percent)}%)`)
+    }
   })
 
   grunt.registerTask('local_tv4_validator_cannot_have_negative_test', 'Check for forbidden negative test folder', function () {
@@ -1151,7 +1175,7 @@ module.exports = function (grunt) {
         } = getOption(callbackParameter.jsonName)
 
         // select the correct AJV object for this schema
-        mainSchema = JSON.parse(callbackParameter.rawFile)
+        mainSchema = callbackParameter.jsonObj
         const versionObj = schemaVersion.getObj(mainSchema)
 
         // External schema present to be included?
@@ -1207,10 +1231,10 @@ module.exports = function (grunt) {
         if (isThisWithExternalSchema) {
           // Must use the root $id/id to call the correct schema JavaScript code
           const validateRootSchema = validations[mainSchemaJsonId]
-          validateRootSchema?.(JSON.parse(callbackParameter.rawFile))
+          validateRootSchema?.(callbackParameter.jsonObj)
         } else {
           // Single schema does not need $id
-          validations(JSON.parse(callbackParameter.rawFile))
+          validations(callbackParameter.jsonObj)
         }
       }
 
@@ -1248,7 +1272,7 @@ module.exports = function (grunt) {
       } = getOption(schemaJsonName)
 
       // select the correct AJV object for this schema
-      const mainSchema = JSON.parse(callbackParameter.rawFile)
+      const mainSchema = callbackParameter.jsonObj
       const versionObj = schemaVersion.getObj(mainSchema)
 
       // Get the correct AJV version

src/package.json

@@ -22,21 +22,22 @@
     "maintenance": "grunt local_maintenance"
   },
   "devDependencies": {
+    "@ltd/j-toml": "^1.30.0",
     "ajv": "^8.11.0",
     "ajv-draft-04": "^1.0.0",
     "ajv-formats": "^2.1.1",
     "ajv-formats-draft2019": "^1.6.1",
     "axios": "^0.27.2",
-    "c8": "^7.11.3",
-    "eslint": "^8.15.0",
+    "c8": "^7.12.0",
+    "eslint": "^8.20.0",
     "eslint-config-standard": "^17.0.0",
     "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-promise": "^6.0.0",
     "find-duplicated-property-keys": "^1.2.7",
     "grunt": "^1.5.3",
-    "js-beautify": "^1.14.3",
+    "js-beautify": "^1.14.4",
     "tv4": "^1.3.0",
-    "yaml": "^2.1.0"
+    "yaml": "^2.1.1"
   }
 }

src/test/hemtt-0.6.2/hemtt-test.toml

@@ -0,0 +1,32 @@
+name = "Advanced Banana Environment"
+prefix = "ABE3"
+author = "ACE Mod Team"
+version = "1.0.0.0"
+files = [ "mod.cpp", "logo.paa", "*.dll" ]
+include = [ "./include" ]
+exclude = [ "*.psd", "*.png", "*.tga" ]
+optionals = [ "tracers", "particles" ]
+folder_optionals = false
+skip = [ "hearing", "zeus" ]
+headerexts = [ "author=me" ]
+modname = "my_mod"
+keyname = "my_key"
+signame = "my_custom_name"
+sigversion = 3
+reuse_private_key = false
+postbuild = [ "!buildtime" ]
+prebuild = [ "" ]
+releasebuild = [ "!build" ]
+
+[scripts.build]
+steps_linux = [ "make linux", "cp bin/ release/{{version}}/ -r" ]
+steps_windows = [ "make windows", "copy bin/ release/{{version}}/" ]
+show_output = true
+foreach = false
+parallel = false
+
+[scripts.buildtime]
+steps = [ "echo {{addon}} took {{time}} ms to build." ]
+show_output = true
+foreach = true
+parallel = true