Skip to content

Commit 28978f8

Browse files
lucian-ioanlucian-ioan
authored
[MS365] [One Drive] Add One Drive datastream (#11814)
1 parent 8fdc5dc commit 28978f8

File tree

17 files changed

+612
-12
lines changed

17 files changed

+612
-12
lines changed

‎packages/o365_metrics/_dev/build/docs/README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,11 @@ Once the secret is created and permissions are granted by admin, setup Elastic A
3636

3737
## Metrics
3838

39-
### OutlookActivity
40-
4139
Uses the Office 365 Management Graph API to retrieve metrics from Office 365.
4240

41+
### Outlook Activity
42+
{{fields "outlook_activity"}}
43+
4344

44-
{{fields "outlook_activity"}}
45+
### One Drive Usage
46+
{{fields "onedrive_usage"}}

‎packages/o365_metrics/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "0.1.2"
3+
changes:
4+
- description: Add `onedrive_usage` data stream.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/11814
27
- version: "0.1.0"
38
changes:
49
- description: Initial draft of the o365_metrics package with the `outlook_activity` data stream.

‎packages/o365_metrics/data_stream/onedrive_usage/_dev/test/pipeline/test-onedrive-usageaccountcounts.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"events": [
3+
{
4+
"onedrive_usageaccountcounts": "{\"Active\":\"0\",\"Report Date\":\"2024-11-23\",\"Report Period\":\"7\",\"Site Type\":\"All\",\"Total\":\"18\",\"Report Refresh Date\":\"2024-11-29\"}"
5+
}
6+
]
7+
}

‎packages/o365_metrics/data_stream/onedrive_usage/_dev/test/pipeline/test-onedrive-usageaccountcounts.json-expected.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"expected": [
3+
{
4+
"ecs": {
5+
"version": "8.16.0"
6+
},
7+
"o365": {
8+
"metrics": {
9+
"onedrive": {
10+
"usage": {
11+
"account": {
12+
"counts": {
13+
"active": {
14+
"count": "0"
15+
},
16+
"report": {
17+
"date": "2024-11-23",
18+
"period": "7",
19+
"refresh_date": "2024-11-29"
20+
},
21+
"total": {
22+
"count": "18"
23+
}
24+
}
25+
}
26+
}
27+
}
28+
}
29+
}
30+
}
31+
]
32+
}

‎packages/o365_metrics/data_stream/onedrive_usage/_dev/test/pipeline/test-onedrive-usagefilecounts.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"events": [
3+
{
4+
"onedrive_usagefilecounts": "{\"Active\":\"0\",\"Report Date\":\"2024-11-23\",\"Report Period\":\"7\",\"Site Type\":\"All\",\"Total\":\"164\",\"Report Refresh Date\":\"2024-11-29\"}"
5+
}
6+
]
7+
}

‎packages/o365_metrics/data_stream/onedrive_usage/_dev/test/pipeline/test-onedrive-usagefilecounts.json-expected.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"expected": [
3+
{
4+
"ecs": {
5+
"version": "8.16.0"
6+
},
7+
"o365": {
8+
"metrics": {
9+
"onedrive": {
10+
"usage": {
11+
"file": {
12+
"counts": {
13+
"active": {
14+
"count": "0"
15+
},
16+
"report": {
17+
"date": "2024-11-23",
18+
"period": "7",
19+
"refresh_date": "2024-11-29"
20+
},
21+
"total": {
22+
"count": "164"
23+
}
24+
}
25+
}
26+
}
27+
}
28+
}
29+
}
30+
}
31+
]
32+
}

‎packages/o365_metrics/data_stream/onedrive_usage/_dev/test/pipeline/test-onedrive-usagestorage.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"events": [
3+
{
4+
"onedrive_usagestorage": "{\"Report Date\":\"2024-11-23\",\"Report Period\":\"7\",\"Site Type\":\"All\",\"Storage Used (Byte)\":\"91659303\",\"Report Refresh Date\":\"2024-11-29\"}"
5+
}
6+
]
7+
}

‎packages/o365_metrics/data_stream/onedrive_usage/_dev/test/pipeline/test-onedrive-usagestorage.json-expected.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"expected": [
3+
{
4+
"ecs": {
5+
"version": "8.16.0"
6+
},
7+
"o365": {
8+
"metrics": {
9+
"onedrive": {
10+
"usage": {
11+
"storage": {
12+
"report": {
13+
"date": "2024-11-23",
14+
"period": "7",
15+
"refresh_date": "2024-11-29"
16+
},
17+
"used_byte": "91659303"
18+
}
19+
}
20+
}
21+
}
22+
}
23+
}
24+
]
25+
}

‎packages/o365_metrics/data_stream/onedrive_usage/agent/stream/cel.yml.hbs‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
config_version: 2
2+
interval: {{interval}}
3+
auth.oauth2:
4+
client.id: {{client_id}}
5+
client.secret: {{client_secret}}
6+
provider: azure
7+
scopes:
8+
{{#each token_scopes as |token_scope|}}
9+
- {{token_scope}}
10+
{{/each}}
11+
endpoint_params:
12+
grant_type: client_credentials
13+
{{#if token_url}}
14+
token_url: {{token_url}}/{{azure_tenant_id}}/oauth2/v2.0/token
15+
{{else if azure_tenant_id}}
16+
azure.tenant_id: {{azure_tenant_id}}
17+
{{/if}}
18+
19+
resource.url: {{url}}
20+
{{#if resource_ssl}}
21+
resource.ssl:
22+
{{resource_ssl}}
23+
{{/if}}
24+
25+
{{#if enable_request_tracer}}
26+
resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson"
27+
{{/if}}
28+
29+
tags:
30+
{{#if preserve_original_event}}
31+
- preserve_original_event
32+
{{/if}}
33+
{{#each tags as |tag|}}
34+
- {{tag}}
35+
{{/each}}
36+
{{#contains "forwarded" tags}}
37+
publisher_pipeline.disable_host: true
38+
{{/contains}}
39+
{{#if processors}}
40+
processors:
41+
{{processors}}
42+
{{/if}}
43+
44+
state:
45+
want_more: false
46+
base:
47+
tenant_id: "{{azure_tenant_id}}"
48+
period: "{{period}}"
49+
50+
redact:
51+
fields:
52+
- base.tenant_id
53+
54+
55+
program: |
56+
state.with(
57+
request(
58+
"GET",
59+
"https://graph.microsoft.com/v1.0/reports/getOneDriveUsageAccountCounts(period='" + state.base.period + "')"
60+
).do_request().as(resp1,
61+
resp1.StatusCode == 200
62+
?
63+
bytes(resp1.Body).mime("text/csv; header=present").as(events1,
64+
request(
65+
"GET",
66+
"https://graph.microsoft.com/v1.0/reports/getOneDriveUsageFileCounts(period='" + state.base.period + "')"
67+
).do_request().as(resp2,
68+
resp2.StatusCode == 200
69+
?
70+
bytes(resp2.Body).mime("text/csv; header=present").as(events2,
71+
request(
72+
"GET",
73+
"https://graph.microsoft.com/v1.0/reports/getOneDriveUsageStorage(period='" + state.base.period + "')"
74+
).do_request().as(resp3,
75+
resp3.StatusCode == 200
76+
?
77+
bytes(resp3.Body).mime("text/csv; header=present").as(events3, {
78+
"events":
79+
events1.map(e, {"onedrive_usageaccountcounts": e.encode_json()}) +
80+
events2.map(e, {"onedrive_usagefilecounts": e.encode_json()}) +
81+
events3.map(e, {"onedrive_usagestorage": e.encode_json()})
82+
})
83+
:
84+
{
85+
"events": {
86+
"error": {
87+
"code": string(resp3.StatusCode),
88+
"id": string(resp3.Status),
89+
"message": "GET:"+(
90+
size(resp3.Body) != 0 ?
91+
string(resp3.Body)
92+
:
93+
string(resp3.Status) + ' (' + string(resp3.StatusCode) + ')'
94+
),
95+
},
96+
},
97+
"want_more": false,
98+
}
99+
)
100+
)
101+
:
102+
{
103+
"events": {
104+
"error": {
105+
"code": string(resp2.StatusCode),
106+
"id": string(resp2.Status),
107+
"message": "GET:"+(
108+
size(resp2.Body) != 0 ?
109+
string(resp2.Body)
110+
:
111+
string(resp2.Status) + ' (' + string(resp2.StatusCode) + ')'
112+
),
113+
},
114+
},
115+
"want_more": false,
116+
}
117+
)
118+
)
119+
:
120+
{
121+
"events": {
122+
"error": {
123+
"code": string(resp1.StatusCode),
124+
"id": string(resp1.Status),
125+
"message": "GET:"+(
126+
size(resp1.Body) != 0 ?
127+
string(resp1.Body)
128+
:
129+
string(resp1.Status) + ' (' + string(resp1.StatusCode) + ')'
130+
),
131+
},
132+
},
133+
"want_more": false,
134+
}
135+
)
136+
)

0 commit comments

Comments
 (0)