Skip to content

Commit

Permalink
updating index to persisted (#131)
Browse files Browse the repository at this point in the history 
Co-authored-by: Mike Cohen <mike@velocidex.com>
  • Loading branch information
@emcgee-work @scudette
emcgee-work and scudette authored Apr 10, 2024
1 parent 2f571e0 commit 6b9587d
Show file tree
Hide file tree
Showing 74 changed files with 474 additions and 571 deletions.
4 changes: 2 additions & 2 deletions crypto/server/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (self *serverPublicKeyResolver) GetPublicKey(

record, err := cvelo_services.GetElasticRecord(
context.Background(), config_obj.OrgId,
"client_keys", client_id)
"persisted", client_id)
if err != nil {
return nil, false
}
Expand Down Expand Up @@ -93,7 +93,7 @@ func (self *serverPublicKeyResolver) SetPublicKey(
EnrollTime: uint64(utils.GetTime().Now().Unix()),
}
return cvelo_services.SetElasticIndex(
self.ctx, config_obj.OrgId, "client_keys", client_id, pem)
self.ctx, config_obj.OrgId, "persisted", client_id, pem)
}

func (self *serverPublicKeyResolver) Clear() {}
Expand Down
8 changes: 4 additions & 4 deletions datastore/datastore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func (self ElasticDatastore) SetSubject(
DocType: "datastore",
Timestamp: utils.GetTime().Now().UnixNano(),
}
return services.SetElasticIndex(self.ctx, config_obj.OrgId, "results", "", record)
return services.SetElasticIndex(self.ctx, config_obj.OrgId, "transient", "", record)
}

func (self ElasticDatastore) SetSubjectWithCompletion(
Expand All @@ -118,15 +118,15 @@ func (self ElasticDatastore) DeleteSubject(

id := services.MakeId(urn.AsClientPath())
return services.DeleteDocumentByQuery(
self.ctx, config_obj.OrgId, "results", json.Format(delete_datastore_doc_query, id), services.SyncDelete)
self.ctx, config_obj.OrgId, "transient", json.Format(delete_datastore_doc_query, id), services.SyncDelete)
}

func (self ElasticDatastore) DeleteSubjectWithCompletion(
config_obj *config_proto.Config,
urn api.DSPathSpec, completion func()) error {
id := services.MakeId(urn.AsClientPath())
return services.DeleteDocumentByQuery(
self.ctx, config_obj.OrgId, "results", json.Format(delete_datastore_doc_query, id), services.AsyncDelete)
self.ctx, config_obj.OrgId, "transient", json.Format(delete_datastore_doc_query, id), services.AsyncDelete)
}

func (self ElasticDatastore) ListChildren(
Expand All @@ -135,7 +135,7 @@ func (self ElasticDatastore) ListChildren(

dir := urn.AsDatastoreDirectory(config_obj)
hits, _, err := services.QueryElasticRaw(self.ctx, config_obj.OrgId,
"results", json.Format(list_children_query, dir))
"transient", json.Format(list_children_query, dir))
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion filestore/s3filestore_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (self *S3FilestoreTest) TestS3FileWriting() {
func TestS3Filestore(t *testing.T) {
suite.Run(t, &S3FilestoreTest{
CloudTestSuite: &testsuite.CloudTestSuite{
Indexes: []string{"clients"},
Indexes: []string{"persisted"},
},
})
}
27 changes: 23 additions & 4 deletions foreman/foreman.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (self Foreman) stopHunt(
`

return cvelo_services.UpdateIndex(
ctx, org_config_obj.OrgId, "hunts", hunt.HuntId, stopHuntQuery)
ctx, org_config_obj.OrgId, "persisted", hunt.HuntId, stopHuntQuery)
}

// Rather than retrieving the entire client record we only get those
Expand Down Expand Up @@ -120,6 +120,12 @@ const getMinimalClientInfoQuery = `{
}
]
}
},
{
"match": {
"doc_type": "clients"
}
}
]
}
Expand All @@ -140,10 +146,11 @@ func getMinimalClientInfo(

result := &api.ClientRecord{
ClientId: client_id,
DocType: "clients",
}
hits, err := cvelo_services.QueryChan(ctx,
config_obj, 1000, config_obj.OrgId,
"clients", query, "client_id")
"persisted", query, "client_id")
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -426,6 +433,12 @@ const (
"gt": %q
}
}
},
{
"match": {
"doc_type": "clients"
}
}
]
}
Expand All @@ -448,6 +461,12 @@ const (
"value": %q
}
}
},
{
"match": {
"doc_type": "clients"
}
}
]
}
Expand All @@ -466,7 +485,7 @@ func (self Foreman) getClientHuntMembership(
seen := make(map[string]bool)
hits, err := cvelo_services.QueryChan(ctx,
config_obj, 1000, config_obj.OrgId,
"clients", query, "client_id")
"persisted", query, "client_id")
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -499,7 +518,7 @@ func (self Foreman) getClientsSeenAfter(

hits, err := cvelo_services.QueryChan(
ctx, config_obj, 1000,
config_obj.OrgId, "clients", query, "ping")
config_obj.OrgId, "persisted", query, "ping")
if err != nil {
logger := logging.GetLogger(config_obj, &logging.FrontendComponent)
logger.Error("getClientsSeenAfter: %v", err)
Expand Down
36 changes: 30 additions & 6 deletions foreman/foreman_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ func (self *ForemanTestSuite) TestClientMonitoring() {
Record: api.ClientRecord{
ClientId: "C.ConnectedClient",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.ConnectedClient_ping",
},
Expand All @@ -128,6 +129,7 @@ func (self *ForemanTestSuite) TestClientMonitoring() {
Record: api.ClientRecord{
ClientId: "C.WithLabel1",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.WithLabel1_ping",
},
Expand All @@ -136,6 +138,7 @@ func (self *ForemanTestSuite) TestClientMonitoring() {
ClientId: "C.WithLabel1",
Labels: []string{"Label1"},
LowerLabels: []string{"label1"},
DocType: "clients",
},
Id: "C.WithLabel1_labels",
},
Expand All @@ -144,6 +147,7 @@ func (self *ForemanTestSuite) TestClientMonitoring() {
Record: api.ClientRecord{
ClientId: "C.WithLabel2",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.WithLabel2_ping",
},
Expand All @@ -152,6 +156,7 @@ func (self *ForemanTestSuite) TestClientMonitoring() {
ClientId: "C.WithLabel2",
Labels: []string{"Label2"},
LowerLabels: []string{"label2"},
DocType: "clients",
},
Id: "C.WithLabel2_labels",
},
Expand All @@ -160,6 +165,7 @@ func (self *ForemanTestSuite) TestClientMonitoring() {
Record: api.ClientRecord{
ClientId: "C.WithLabel1And2",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.WithLabel1And2_ping",
},
Expand All @@ -168,6 +174,7 @@ func (self *ForemanTestSuite) TestClientMonitoring() {
ClientId: "C.WithLabel1And2",
Labels: []string{"Label1", "Label2"},
LowerLabels: []string{"label1", "label2"},
DocType: "clients",
},
Id: "C.WithLabel1And2_labels",
},
Expand All @@ -177,6 +184,7 @@ func (self *ForemanTestSuite) TestClientMonitoring() {
Record: api.ClientRecord{
ClientId: "C.OfflineClient",
Ping: uint64(utils.GetTime().Now().Add(-72 * time.Hour).UnixNano()),
DocType: "clients",
},
Id: "C.OfflineClient_ping",
},
Expand All @@ -185,7 +193,7 @@ func (self *ForemanTestSuite) TestClientMonitoring() {
// Add these clients directly into the index.
for _, c := range clients {
err := cvelo_services.SetElasticIndex(
self.Ctx, config_obj.OrgId, "clients", c.Id, c.Record)
self.Ctx, config_obj.OrgId, "persisted", c.Id, c.Record)
assert.NoError(self.T(), err)
}

Expand Down Expand Up @@ -370,6 +378,7 @@ func (self *ForemanTestSuite) TestHuntsAllClients() {
Record: api.ClientRecord{
ClientId: "C.ConnectedClient",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.ConnectedClient_ping",
},
Expand All @@ -380,20 +389,23 @@ func (self *ForemanTestSuite) TestHuntsAllClients() {
Record: api.ClientRecord{
ClientId: "C.AlreadyRanAllClients",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.AlreadyRanAllClients_ping",
},
{
Record: api.ClientRecord{
ClientId: "C.AlreadyRanAllClients",
AssignedHunts: []string{"H.AllClients"},
DocType: "clients",
},
Id: "C.AlreadyRanAllClients_hunts",
},
{
Record: api.ClientRecord{
ClientId: "C.WithLabelFoo",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.WithLabelFoo_ping",
},
Expand All @@ -402,6 +414,7 @@ func (self *ForemanTestSuite) TestHuntsAllClients() {
ClientId: "C.WithLabelFoo",
Labels: []string{"Foo"},
LowerLabels: []string{"foo"},
DocType: "clients",
},
Id: "C.WithLabelFoo_labels",
},
Expand All @@ -411,6 +424,7 @@ func (self *ForemanTestSuite) TestHuntsAllClients() {
Record: api.ClientRecord{
ClientId: "C.OfflineClient",
Ping: uint64(utils.GetTime().Now().Add(-72 * time.Hour).UnixNano()),
DocType: "clients",
},
Id: "C.OfflineClient_ping",
},
Expand All @@ -419,7 +433,7 @@ func (self *ForemanTestSuite) TestHuntsAllClients() {
// Add these clients directly into the index.
for _, c := range clients {
err := cvelo_services.SetElasticIndex(
self.Ctx, config_obj.OrgId, "clients", c.Id, c.Record)
self.Ctx, config_obj.OrgId, "persisted", c.Id, c.Record)
assert.NoError(self.T(), err)
}

Expand Down Expand Up @@ -514,11 +528,12 @@ func (self *ForemanTestSuite) TestHuntsAllClients() {
// Update the ping time - the client will only be scheduled once
// it is online.
err = cvelo_services.SetElasticIndex(self.Ctx,
config_obj.OrgId, "clients", "C.ConnectedClient_ping",
config_obj.OrgId, "persisted", "C.ConnectedClient_ping",
&api.ClientRecord{
ClientId: "C.ConnectedClient",
Type: "ping",
Ping: uint64(utils.GetTime().Now().Add(time.Second).UnixNano()),
DocType: "clients",
})
assert.NoError(self.T(), err)

Expand Down Expand Up @@ -554,9 +569,10 @@ func (self *ForemanTestSuite) testHuntsExpireInFuture() {
AssignedHunts: []string{},
Labels: []string{},
LowerLabels: []string{},
DocType: "clients",
}
err := cvelo_services.SetElasticIndex(
self.Ctx, config_obj.OrgId, "clients", c.ClientId, c)
self.Ctx, config_obj.OrgId, "persisted", c.ClientId, c)
assert.NoError(self.T(), err)

plan, err := NewPlan(config_obj)
Expand Down Expand Up @@ -663,13 +679,15 @@ func (self *ForemanTestSuite) TestHuntsByOS() {
Record: api.ClientRecord{
ClientId: "C.Windows1",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.Windows1_ping",
},
{
Record: api.ClientRecord{
ClientId: "C.Windows1",
System: "windows",
DocType: "clients",
},
Id: "C.Windows1",
},
Expand All @@ -678,41 +696,47 @@ func (self *ForemanTestSuite) TestHuntsByOS() {
Record: api.ClientRecord{
ClientId: "C.Windows2",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.Windows2_ping",
},
{
Record: api.ClientRecord{
ClientId: "C.Windows2",
System: "windows",
DocType: "clients",
},
Id: "C.Windows2",
},
{
Record: api.ClientRecord{
ClientId: "C.Linux1",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.Linux1_ping",
},
{
Record: api.ClientRecord{
ClientId: "C.Linux1",
System: "linux",
DocType: "clients",
},
Id: "C.Linux1",
},
{
Record: api.ClientRecord{
ClientId: "C.Linux2",
Ping: uint64(utils.GetTime().Now().UnixNano()),
DocType: "clients",
},
Id: "C.Linux2_ping",
},
{
Record: api.ClientRecord{
ClientId: "C.Linux2",
System: "linux",
DocType: "clients",
},
Id: "C.Linux2",
},
Expand All @@ -721,7 +745,7 @@ func (self *ForemanTestSuite) TestHuntsByOS() {
// Add these clients directly into the index.
for _, c := range clients {
err := cvelo_services.SetElasticIndex(
self.Ctx, config_obj.OrgId, "clients", c.Id, c.Record)
self.Ctx, config_obj.OrgId, "persisted", c.Id, c.Record)
assert.NoError(self.T(), err)
}

Expand Down Expand Up @@ -796,7 +820,7 @@ func (self *ForemanTestSuite) checkAssignedHunts(clientId string, expectedHunts
func TestForeman(t *testing.T) {
suite.Run(t, &ForemanTestSuite{
CloudTestSuite: &testsuite.CloudTestSuite{
Indexes: []string{"clients", "hunts", "repository", "tasks"},
Indexes: []string{"persisted"},
},
})
}
Expand Down
Loading

0 comments on commit 6b9587d

Please sign in to comment.