Convert code to streaming interface for all APIs; add benchmark testing

dmwm · Dec 7, 2020 · 9f6788b · 9f6788b
1 parent 0c3c415
commit 9f6788b
Show file tree

Hide file tree

Showing 18 changed files with 188 additions and 377 deletions.
diff --git a/Makefile b/Makefile
@@ -33,7 +33,9 @@ install:
 clean:
 	go clean; rm -rf pkg
 
-test : test1
+test : test1 bench
 
 test1:
 	cd test; go test -v .
+bench:
+	cd test; go test -bench=.
diff --git a/dbs/acquisitioneras.go b/dbs/acquisitioneras.go
@@ -1,11 +1,13 @@
 package dbs
 
 import (
+	"errors"
 	"fmt"
+	"net/http"
 )
 
 // acquisitioneras API
-func (API) AcquisitionEras(params Record) []Record {
+func (API) AcquisitionEras(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := "WHERE "
@@ -14,7 +16,7 @@ func (API) AcquisitionEras(params Record) []Record {
 	acquisitioneras := getValues(params, "data_tier_name")
 	if len(acquisitioneras) > 1 {
 		msg := "The acquisitioneras API does not support list of acquisitioneras"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(acquisitioneras) == 1 {
 		op, val := opVal(acquisitioneras[0])
 		cond := fmt.Sprintf(" AE.ACQUISITION_ERA_NAME %s %s", op, placeholder("acquisition_era_name"))
@@ -26,5 +28,5 @@ func (API) AcquisitionEras(params Record) []Record {
 	// get SQL statement from static area
 	stm := getSQL("acquisitioneras")
 	// use generic query API to fetch the results from DB
-	return executeAll(stm+where, args...)
+	return executeAll(w, stm+where, args...)
 }
diff --git a/dbs/blocks.go b/dbs/blocks.go
@@ -1,12 +1,14 @@
 package dbs
 
 import (
+	"errors"
 	"fmt"
+	"net/http"
 	"strings"
 )
 
 // blocks API
-func (API) Blocks(params Record) []Record {
+func (API) Blocks(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := "WHERE "
@@ -15,7 +17,7 @@ func (API) Blocks(params Record) []Record {
 	blocks := getValues(params, "block_name")
 	if len(blocks) > 1 {
 		msg := "Unsupported list of blocks"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(blocks) == 1 {
 		op, val := opVal(blocks[0])
 		cond := fmt.Sprintf(" B.BLOCK_NAME %s %s", op, placeholder("block_name"))
@@ -25,7 +27,7 @@ func (API) Blocks(params Record) []Record {
 	datasets := getValues(params, "dataset")
 	if len(datasets) > 1 {
 		msg := "The files API does not support list of datasets"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(datasets) == 1 {
 		op, val := opVal(datasets[0])
 		cond := fmt.Sprintf(" DS.DATASET %s %s", op, placeholder("dataset"))
@@ -35,11 +37,11 @@ func (API) Blocks(params Record) []Record {
 	// get SQL statement from static area
 	stm := getSQL("blocks")
 	// use generic query API to fetch the results from DB
-	return executeAll(stm+where, args...)
+	return executeAll(w, stm+where, args...)
 }
 
 // blockparent API
-func (API) BlockParent(params Record) []Record {
+func (API) BlockParent(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := "WHERE "
@@ -48,7 +50,7 @@ func (API) BlockParent(params Record) []Record {
 	blockparent := getValues(params, "block_name")
 	if len(blockparent) > 1 {
 		msg := "Unsupported list of blockparent"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(blockparent) == 1 {
 		op, val := opVal(blockparent[0])
 		cond := fmt.Sprintf(" BP.BLOCK_NAME %s %s", op, placeholder("block_name"))
@@ -58,11 +60,11 @@ func (API) BlockParent(params Record) []Record {
 	// get SQL statement from static area
 	stm := getSQL("blockparent")
 	// use generic query API to fetch the results from DB
-	return executeAll(stm+where, args...)
+	return executeAll(w, stm+where, args...)
 }
 
 // blockchildren API
-func (API) BlockChildren(params Record) []Record {
+func (API) BlockChildren(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := "WHERE "
@@ -71,7 +73,7 @@ func (API) BlockChildren(params Record) []Record {
 	blockchildren := getValues(params, "block_name")
 	if len(blockchildren) > 1 {
 		msg := "Unsupported list of blockchildren"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(blockchildren) == 1 {
 		op, val := opVal(blockchildren[0])
 		cond := fmt.Sprintf(" BP.BLOCK_NAME %s %s", op, placeholder("block_name"))
@@ -81,11 +83,11 @@ func (API) BlockChildren(params Record) []Record {
 	// get SQL statement from static area
 	stm := getSQL("blockchildren")
 	// use generic query API to fetch the results from DB
-	return executeAll(stm+where, args...)
+	return executeAll(w, stm+where, args...)
 }
 
 // blocksummaries API
-func (API) BlockSummaries(params Record) []Record {
+func (API) BlockSummaries(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var stm, where_clause string
 	var args []interface{}
@@ -113,7 +115,7 @@ func (API) BlockSummaries(params Record) []Record {
 	dataset := getValues(params, "dataset")
 	if len(dataset) > 1 {
 		msg := "Unsupported list of dataset"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(dataset) == 1 {
 		_, val := opVal(dataset[0])
 		args = append(args, val)
@@ -124,11 +126,11 @@ func (API) BlockSummaries(params Record) []Record {
 		stm = strings.Replace(stm, "where_clause", where_clause, -1)
 	}
 	// use generic query API to fetch the results from DB
-	return executeAll(genSQL+stm, args...)
+	return executeAll(w, genSQL+stm, args...)
 }
 
 // blockorigin API
-func (API) BlockOrigin(params Record) []Record {
+func (API) BlockOrigin(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := "WHERE "
@@ -137,7 +139,7 @@ func (API) BlockOrigin(params Record) []Record {
 	block := getValues(params, "block_name")
 	if len(block) > 1 {
 		msg := "Unsupported list of block"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(block) == 1 {
 		op, val := opVal(block[0])
 		cond := fmt.Sprintf(" B.BLOCK_NAME %s %s", op, placeholder("block_name"))
@@ -147,7 +149,7 @@ func (API) BlockOrigin(params Record) []Record {
 	dataset := getValues(params, "dataset")
 	if len(dataset) > 1 {
 		msg := "Unsupported list of dataset"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(dataset) == 1 {
 		op, val := opVal(dataset[0])
 		cond := fmt.Sprintf(" DS.DATASET %s %s", op, placeholder("dataset"))
@@ -157,5 +159,5 @@ func (API) BlockOrigin(params Record) []Record {
 	// get SQL statement from static area
 	stm := getSQL("blockorigin")
 	// use generic query API to fetch the results from DB
-	return executeAll(stm+where, args...)
+	return executeAll(w, stm+where, args...)
 }
diff --git a/dbs/datasets.go b/dbs/datasets.go
@@ -2,13 +2,14 @@ package dbs
 
 import (
 	"database/sql"
+	"errors"
 	"fmt"
 	"net/http"
 	"strings"
 )
 
 // Datasets API
-func (API) Datasets(params Record) []Record {
+func (API) Datasets(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := "WHERE "
@@ -60,11 +61,11 @@ func (API) Datasets(params Record) []Record {
 		vals = []interface{}{new(sql.NullString)}
 	}
 	// use generic query API to fetch the results from DB
-	return execute(genSQL+stm+where, cols, vals, args...)
+	return execute(w, genSQL+stm+where, cols, vals, args...)
 }
 
 // datasetparent API
-func (API) DatasetParent(params Record) []Record {
+func (API) DatasetParent(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := "WHERE "
@@ -73,24 +74,24 @@ func (API) DatasetParent(params Record) []Record {
 	datasetparent := getValues(params, "dataset")
 	if len(datasetparent) > 1 {
 		msg := "The datasetparent API does not support list of datasetparent"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(datasetparent) == 1 {
 		op, val := opVal(datasetparent[0])
 		cond := fmt.Sprintf(" D.DATASET %s %s", op, placeholder("dataset"))
 		where += addCond(where, cond)
 		args = append(args, val)
 	} else {
 		msg := fmt.Sprintf("No arguments for datasetparent API")
-		return errorRecord(msg)
+		return errors.New(msg)
 	}
 	// get SQL statement from static area
 	stm := getSQL("datasetparent")
 	// use generic query API to fetch the results from DB
-	return executeAll(stm+where, args...)
+	return executeAll(w, stm+where, args...)
 }
 
 // datasetchildren API
-func (API) DatasetChildren(params Record) []Record {
+func (API) DatasetChildren(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := "WHERE "
@@ -99,24 +100,24 @@ func (API) DatasetChildren(params Record) []Record {
 	datasetchildren := getValues(params, "dataset")
 	if len(datasetchildren) > 1 {
 		msg := "The datasetchildren API does not support list of datasetchildren"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(datasetchildren) == 1 {
 		op, val := opVal(datasetchildren[0])
 		cond := fmt.Sprintf(" D.DATASET %s %s", op, placeholder("dataset"))
 		where += addCond(where, cond)
 		args = append(args, val)
 	} else {
 		msg := fmt.Sprintf("No arguments for datasetchildren API")
-		return errorRecord(msg)
+		return errors.New(msg)
 	}
 	// get SQL statement from static area
 	stm := getSQL("datasetchildren")
 	// use generic query API to fetch the results from DB
-	return executeAll(stm+where, args...)
+	return executeAll(w, stm+where, args...)
 }
 
 // datasetaccesstypes API
-func (API) DatasetAccessTypes(params Record) []Record {
+func (API) DatasetAccessTypes(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := " WHERE "
@@ -125,7 +126,7 @@ func (API) DatasetAccessTypes(params Record) []Record {
 	datasetaccesstypes := getValues(params, "dataset_access_type")
 	if len(datasetaccesstypes) > 1 {
 		msg := "The datasetaccesstypes API does not support list of datasetaccesstypes"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(datasetaccesstypes) == 1 {
 		op, val := opVal(datasetaccesstypes[0])
 		cond := fmt.Sprintf(" DT.DATASET_ACCESS_TYPE %s %s", op, placeholder("dataset_access_type"))
@@ -137,61 +138,5 @@ func (API) DatasetAccessTypes(params Record) []Record {
 	// get SQL statement from static area
 	stm := getSQL("datasetaccesstypes")
 	// use generic query API to fetch the results from DB
-	return executeAll(stm+where, args...)
-}
-
-// Datasets API
-func (API) DatasetsNew(params Record, w http.ResponseWriter) error {
-	// variables we'll use in where clause
-	var args []interface{}
-	where := "WHERE "
-
-	// parse detail arugment
-	detail := getSingleValue(params, "detail")
-
-	// parse is_dataset_valid argument
-	isValid := getSingleValue(params, "is_dataset_valid")
-	if isValid == "" {
-		isValid = "1"
-	}
-	where += fmt.Sprintf("D.IS_DATASET_VALID = %s", placeholder("is_dataset_valid"))
-	args = append(args, isValid)
-
-	// parse dataset_id argument
-	dataset_access_type := getSingleValue(params, "dataset_access_type")
-	if dataset_access_type == "" {
-		dataset_access_type = "VALID"
-	}
-	where += fmt.Sprintf(" AND DP.DATASET_ACCESS_TYPE = %s", placeholder("dataset_access_type"))
-	args = append(args, dataset_access_type)
-
-	// parse dataset argument
-	datasets := getValues(params, "dataset")
-	genSQL := ""
-	if len(datasets) > 1 {
-		where += fmt.Sprintf(" AND D.DATASET in (SELECT TOKEN FROM TOKEN_GENERATOR)")
-		where += fmt.Sprintf(" AND DP.DATASET_ACCESS_TYPE = %s", placeholder("dataset_access_type"))
-		var vals []string
-		genSQL, vals = tokens(datasets)
-		for _, d := range vals {
-			args = append(args, d, d, d) // append three values since tokens generates placeholders for them
-		}
-	} else if len(datasets) == 1 {
-		op, val := opVal(datasets[0])
-		where += fmt.Sprintf(" AND D.DATASET %s %s", op, placeholder("dataset"))
-		where += fmt.Sprintf(" AND DP.DATASET_ACCESS_TYPE = %s", placeholder("dataset_access_type"))
-		args = append(args, val)
-	}
-
-	// get SQL statement from static area
-	stm := getSQL("datasets")
-	cols := []string{"dataset_id", "dataset", "prep_id", "xtcrosssection", "creation_date", "create_by", "last_modification_date", "last_modified_by", "primary_ds_name", "primary_ds_type", "processed_ds_name", "data_tier_name", "dataset_access_type", "acquisition_era_name", "processing_version", "physics_group_name"}
-	vals := []interface{}{new(sql.NullInt64), new(sql.NullString), new(sql.NullString), new(sql.NullFloat64), new(sql.NullInt64), new(sql.NullString), new(sql.NullInt64), new(sql.NullString), new(sql.NullString), new(sql.NullString), new(sql.NullString), new(sql.NullString), new(sql.NullString), new(sql.NullString), new(sql.NullInt64), new(sql.NullString)}
-	if strings.ToLower(detail) != "true" {
-		stm = getSQL("datasets_short")
-		cols = []string{"dataset"}
-		vals = []interface{}{new(sql.NullString)}
-	}
-	// use generic query API to fetch the results from DB
-	return executeNew(w, genSQL+stm+where, cols, vals, args...)
+	return executeAll(w, stm+where, args...)
 }
diff --git a/dbs/datatypes.go b/dbs/datatypes.go
@@ -1,11 +1,13 @@
 package dbs
 
 import (
+	"errors"
 	"fmt"
+	"net/http"
 )
 
 // datatypes API
-func (API) DataTypes(params Record) []Record {
+func (API) DataTypes(params Record, w http.ResponseWriter) error {
 	// variables we'll use in where clause
 	var args []interface{}
 	where := "WHERE "
@@ -14,7 +16,7 @@ func (API) DataTypes(params Record) []Record {
 	datatypes := getValues(params, "datatype")
 	if len(datatypes) > 1 {
 		msg := "The datatypes API does not support list of datatypes"
-		return errorRecord(msg)
+		return errors.New(msg)
 	} else if len(datatypes) == 1 {
 		op, val := opVal(datatypes[0])
 		cond := fmt.Sprintf(" DT.datatype %s %s", op, placeholder("datatype"))
@@ -26,5 +28,5 @@ func (API) DataTypes(params Record) []Record {
 	// get SQL statement from static area
 	stm := getSQL("datatypes")
 	// use generic query API to fetch the results from DB
-	return executeAll(stm+where, args...)
+	return executeAll(w, stm+where, args...)
 }