Revert backend performance optimisations

Makefile

@@ -4,7 +4,7 @@ OVERRIDE_FILE=./Docker/config/local_override.json
 BINARY=./output/cvelociraptor
 CONFIG_ARGS= --config $(SERVER_CONFIG) --override_file $(OVERRIDE_FILE)
 CLIENT_CONFIG_ARGS= --config $(CLIENT_CONFIG) --override_file $(OVERRIDE_FILE)
-DLV=dlv debug --init ./scripts/dlv.init --build-flags="-tags 'server_vql extras'" ./bin/ -- --debug --debug_filter result_set
+DLV=dlv debug --build-flags="-tags 'server_vql extras'" ./bin/ --
 WRITEBACK_DIR=/tmp/pool_writebacks/
 POOL_NUMBER=20
 
@@ -31,7 +31,7 @@ dump:
 	$(BINARY) $(CONFIG_ARGS) elastic dump -v
 
 dump_persisted:
-	$(BINARY) $(CONFIG_ARGS) elastic dump --index="persisted" -v --dump_count 2000
+	$(BINARY) $(CONFIG_ARGS) elastic dump --index="persisted" -v --org_id O123
 
 debug_gui:
 	$(DLV) $(CONFIG_ARGS) gui -v

bin/debug.go

@@ -22,7 +22,6 @@ import (
 	"log"
 	"net/http"
 	_ "net/http/pprof"
-	"regexp"
 
 	"www.velocidex.com/golang/cloudvelo/services"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
@@ -31,9 +30,7 @@ import (
 )
 
 var (
-	debug_flag  = app.Flag("debug", "Enables debug and profile server.").Bool()
-	debug_regex = app.Flag("debug_filter", "A regex to filter the debug source.").
-			Default(".").String()
+	debug_flag      = app.Flag("debug", "Enables debug and profile server.").Bool()
 	debug_flag_port = app.Flag("debug_port", "Port for the debug server.").
 			Default("6060").Int64()
 )
@@ -43,12 +40,7 @@ func initDebugServer(config_obj *config_proto.Config) error {
 		logger := logging.GetLogger(config_obj, &logging.FrontendComponent)
 		logger.Info("<green>Starting</> debug server on <red>http://127.0.0.1:%v/debug/pprof", *debug_flag_port)
 
-		re, err := regexp.Compile("(?i)" + *debug_regex)
-		if err != nil {
-			return err
-		}
-
-		services.SetDebugLogger(config_obj, re)
+		services.SetDebugLogger(config_obj)
 
 		// Switch off the debug flag so we do not run this again. (The
 		// GUI runs this function multiple times).

config/api.go

@@ -47,14 +47,6 @@ type ElasticConfiguration struct {
 	DedicatedForeman    bool     `json:"dedicated_foreman"`
 	DedicatedForemanOrg string   `json:"dedicated_foreman_org"`
 	ForemanExcludedOrgs []string `json:"foreman_excluded_orgs"`
-
-	// How many rows to collect before flushing into a result set
-	// packet. Default is 100.
-	RowsPerResultSet uint64 `json:"rows_per_result_set"`
-	MaxSizePerPacket uint64 `json:"max_size_per_packet"`
-
-	// Minimum amount of time we cache flow indexes (Default 1)
-	MinFlowCacheTimeMin int64 `json:"min_flow_cache_time_min"`
 }
 
 // Create a new cloud config object which contains the original

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/alecthomas/assert v1.0.0
 	github.com/aws/aws-sdk-go v1.55.6
 	github.com/aws/aws-sdk-go-v2/config v1.27.6
-	github.com/codesoap/lineworker v0.2.1
 	github.com/evanphx/json-patch/v5 v5.9.11
 	github.com/google/uuid v1.6.0
 	github.com/magefile/mage v1.15.0

go.sum

@@ -229,8 +229,6 @@ github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
 github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/codesoap/lineworker v0.2.1 h1:YZI1Xamk7/i9J3iPWz7hopttZKwFEn0GXOsF5UeebNI=
-github.com/codesoap/lineworker v0.2.1/go.mod h1:aatQ4DVq3bEOUG0O5lAcBhSiwH2BmwmSYQPJjrYCv6w=
 github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI=
 github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

ingestion/fixtures/TestListDirectory.golden

@@ -86,38 +86,6 @@
    "doc_type": "vfs",
    "doc_id": "1cbcacc4b30a75913df3a051a73742ca2bca1963",
    "timestamp": 1661385600000000000
-  },
-  {
-   "timestamp": 1661385600000000000,
-   "vfs_path": "/clients/C.77ad4285690698d9/artifacts/System.VFS.ListDirectory/F.CEV6I8LHAT83O/Listing.json",
-   "id": "",
-   "end_row": 1,
-   "total_rows": 0,
-   "type": "rs_metadata"
-  },
-  {
-   "timestamp": 1661385600000000000,
-   "vfs_path": "/clients/C.77ad4285690698d9/artifacts/System.VFS.ListDirectory/F.CEV6I8LHAT83O/Stats.json",
-   "id": "",
-   "end_row": 1,
-   "total_rows": 0,
-   "type": "rs_metadata"
-  },
-  {
-   "timestamp": 1661385600000000000,
-   "vfs_path": "/clients/C.77ad4285690698d9/collections/F.CEV6I8LHAT83O/logs.json",
-   "id": "",
-   "end_row": 14,
-   "total_rows": 0,
-   "type": "rs_metadata"
-  },
-  {
-   "timestamp": 1661385600000000000,
-   "vfs_path": "/clients/C.77ad4285690698d9/collections/F.CEV6I8LHAT83O/logs.json",
-   "id": "",
-   "end_row": 5,
-   "total_rows": 0,
-   "type": "rs_metadata"
   }
  ],
  "System.VFS.ListDirectory vfs": [

result_sets/simple/factory.go

@@ -6,22 +6,12 @@ import (
 	"time"
 
 	"www.velocidex.com/golang/cloudvelo/filestore"
-	cvelo_services "www.velocidex.com/golang/cloudvelo/services"
 	"www.velocidex.com/golang/velociraptor/file_store/api"
 	"www.velocidex.com/golang/velociraptor/json"
 	"www.velocidex.com/golang/velociraptor/result_sets"
 	"www.velocidex.com/golang/velociraptor/utils"
 )
 
-/*
-  Since we write result sets on the transient index we can not
-  actually delete anything.
-
-  Therefore we end up writing a new ID to indicate a new result set
-  and we write a metadata record to point the result set at the latest
-  ID.
-*/
-
 type ResultSetFactory struct{}
 
 func (self ResultSetFactory) NewResultSetWriter(
@@ -31,19 +21,7 @@ func (self ResultSetFactory) NewResultSetWriter(
 	completion func(),
 	truncate result_sets.WriteMode) (result_sets.ResultSetWriter, error) {
 
-	cvelo_services.Count("NewResultSetWriter")
-
 	cloud_config_obj := filestore.GetConfigObj(file_store_factory)
-	rows_per_result_set := cloud_config_obj.Cloud.RowsPerResultSet
-	if rows_per_result_set == 0 {
-		rows_per_result_set = 1000
-	}
-
-	max_size_per_packet := cloud_config_obj.Cloud.MaxSizePerPacket
-	if max_size_per_packet == 0 {
-		max_size_per_packet = 1024 * 1024
-	}
-
 	config_obj := cloud_config_obj.VeloConf()
 
 	new_id := fmt.Sprintf("%v", utils.GetGUID())
@@ -55,7 +33,6 @@ func (self ResultSetFactory) NewResultSetWriter(
 		Timestamp: utils.GetTime().Now().UnixNano(),
 		VFSPath:   base_record.VFSPath,
 		ID:        new_id,
-		EndRow:    0,
 		Type:      "rs_metadata",
 	}
 
@@ -76,26 +53,19 @@ func (self ResultSetFactory) NewResultSetWriter(
 	}
 
 	return &ElasticSimpleResultSetWriter{
-		org_id:              utils.GetOrgId(config_obj),
-		config_obj:          config_obj,
-		log_path:            log_path,
-		opts:                opts,
-		ctx:                 context.Background(),
-		sync:                utils.CompareFuncs(completion, utils.SyncCompleter),
-		version:             md.ID,
-		md:                  md,
-		start_row:           md.EndRow,
-		rows_per_result_set: rows_per_result_set,
-		max_size_per_packet: max_size_per_packet,
+		org_id:   utils.GetOrgId(config_obj),
+		log_path: log_path,
+		opts:     opts,
+		ctx:      context.Background(),
+		sync:     utils.CompareFuncs(completion, utils.SyncCompleter),
+		version:  md.ID,
 	}, nil
 }
 
 func (self ResultSetFactory) NewResultSetReader(
 	file_store_factory api.FileStore,
 	log_path api.FSPathSpec) (result_sets.ResultSetReader, error) {
 
-	cvelo_services.Count("NewResultSetReader")
-
 	ctx := context.Background()
 
 	cloud_config_obj := filestore.GetConfigObj(file_store_factory)
@@ -107,29 +77,10 @@ func (self ResultSetFactory) NewResultSetReader(
 		return nil, utils.NotFoundError
 	}
 
-	// This signifies that the result set is incomplete - we can not
-	// open it. It happens when we abort the writing of the result set
-	// prematurely.
-	if existing_md.TotalRows < 0 {
-		return nil, utils.NotFoundError
-	}
-
-	base_record := NewSimpleResultSetRecord(log_path, existing_md.ID)
-
-	// Backwards compatibility - should not be needed with newer results
-	if existing_md.EndRow == 0 {
-		org_id := filestore.GetOrgId(file_store_factory)
-		last_rec, err := getLastRecord(ctx, org_id, base_record)
-		if err == nil {
-			existing_md.EndRow = last_rec.EndRow
-		}
-	}
-
 	return &SimpleResultSetReader{
 		file_store_factory: file_store_factory,
 		log_path:           log_path,
 		mtime:              time.Unix(0, existing_md.Timestamp),
-		base_record:        base_record,
-		md:                 existing_md,
+		base_record:        NewSimpleResultSetRecord(log_path, existing_md.ID),
 	}, nil
 }

result_sets/simple/metadata.go

@@ -12,7 +12,6 @@ import (
 )
 
 const (
-	// Get the latest metadata record with the most recent timestamp.
 	result_md_query = `
 {
  "query": {"bool": {"must": [
@@ -29,8 +28,6 @@ type ResultSetMetadataRecord struct {
 	Timestamp int64  `json:"timestamp"`
 	VFSPath   string `json:"vfs_path"`
 	ID        string `json:"id"`
-	EndRow    int64  `json:"end_row"`
-	TotalRows int64  `json:"total_rows"`
 	Type      string `json:"type"`
 }
 
@@ -44,11 +41,6 @@ func GetResultSetMetadata(
 	config_obj *config_proto.Config,
 	log_path api.FSPathSpec) (*ResultSetMetadataRecord, error) {
 
-	cvelo_services.Count("GetResultSetMetadata")
-	cvelo_services.Debug(
-		cvelo_services.DEBUG_RESULT_SET,
-		"GetResultSetMetadata: %v", log_path.AsClientPath())()
-
 	base_record := NewSimpleResultSetRecord(log_path, "")
 
 	query := json.Format(result_md_query, base_record.VFSPath)

result_sets/simple/reader.go

@@ -43,7 +43,6 @@ type SimpleResultSetReader struct {
 	opts               result_sets.ResultSetOptions
 	base_record        *SimpleResultSetRecord
 	mtime              time.Time
-	md                 *ResultSetMetadataRecord
 
 	stacker api.FSPathSpec
 }
@@ -61,11 +60,6 @@ func (self *SimpleResultSetReader) SeekToRow(start int64) error {
 func (self *SimpleResultSetReader) getPacket(
 	ctx context.Context, row int64) (*SimpleResultSetRecord, error) {
 
-	cvelo_services.Count("SimpleResultSet: getPacket")
-	cvelo_services.Debug(
-		cvelo_services.DEBUG_RESULT_SET, "getPacket: %v (%v)",
-		self.log_path, row)()
-
 	var artifact_clause, query string
 
 	if self.base_record.VFSPath != "" {
@@ -137,11 +131,6 @@ func (self *SimpleResultSetReader) Rows(
 				break
 			}
 
-			// We read enough rows
-			if self.md.EndRow > 0 && self.row >= self.md.EndRow {
-				break
-			}
-
 			packet, err := self.getPacket(ctx, self.row)
 			if err != nil {
 				return
@@ -201,11 +190,6 @@ func (self *SimpleResultSetReader) JSON(
 				break
 			}
 
-			// We read enough rows
-			if self.md.EndRow > 0 && self.row >= self.md.EndRow {
-				break
-			}
-
 			packet, err := self.getPacket(ctx, self.row)
 			if err != nil {
 				return
@@ -260,21 +244,20 @@ func (self *SimpleResultSetReader) Close() {}
 
 // Figure out how many rows are in this collection in total.
 func (self *SimpleResultSetReader) TotalRows() int64 {
-	return self.md.EndRow
+	org_id := filestore.GetOrgId(self.file_store_factory)
+	last_rec, err := getLastRecord(org_id, self.base_record)
+	if err != nil {
+		return -1
+	}
+
+	return last_rec.EndRow
 }
 
-// Deprecated: Current code sets the last row number in another
-// metadata document in the EndRow field. This is here for backwards
-// compatibility.
-func getLastRecord(
-	ctx context.Context,
-	org_id string,
+func getLastRecord(org_id string,
 	base_record *SimpleResultSetRecord) (*SimpleResultSetRecord, error) {
-
+	ctx := context.Background()
 	var artifact_clause, query string
 
-	cvelo_services.Count("SimpleResultSet: getLastRecord")
-
 	if base_record.VFSPath != "" {
 		query = json.Format(`
 {"sort": {"end_row": {"order": "desc"}},