Skip to content

Commit

Permalink
Merge pull request #95 from apdibbo/more_accounting
Browse files Browse the repository at this point in the history 
Improve accounting scripts
  • Loading branch information
@DavidFair
DavidFair authored Nov 21, 2023
2 parents db94762 + d0be537 commit 7708756
Show file tree
Hide file tree
Showing 16 changed files with 657 additions and 175 deletions.
2 changes: 2 additions & 0 deletions OpenStack-accounting/etc/thecount/thecount.conf.example
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[database]
connection =mysql+pymysql://<dbusername>:<dbpassword>@<dbhost>:3306
23 changes: 23 additions & 0 deletions OpenStack-accounting/readme.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# TheCount


## Prerequisites

The following packages are required:
- python3-requests
- python36-sqlalchemy
- python36-PyMySQL

A database user with read access to the relevant databases for each OpenStack component you are accounting for

## Installation

Copy the scripts into `/usr/local/sbin` as shown in this repo
Create a config file with a database connection string in the format shown in `/etc/thecount/thecount.conf.example` in `/etc/thecount/thecount.conf`
Create the stored procedures in the `sql` directory appropriate db.

## Use

`now-accounting.sh` generates accounting for the last 24 hours - recommend setting up a cron to run this at midnight
`past-accounting.sh` takes a start date and an end date in the format "%Y-%m-%d %H:%M" to generate accounting for past usage where possible
`*-extract_accounting.py` takes a start date and an end date in the format "%Y-%m-%d %H:%M" to generate accounting for that component
3 changes: 3 additions & 0 deletions OpenStack-accounting/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
python3-requests
python36-sqlalchemy
python36-PyMySQL
44 changes: 44 additions & 0 deletions OpenStack-accounting/sql/cinder_get_accounting_data.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime)
BEGIN
/*
This procedure generates accounting data for cinder
*/
SELECT
IFNULL(v.availability_zone, 'nova') AS AvailabilityZone,
p.name AS Project,
pp.name AS Department,
COUNT(v.id) AS Volumes,
"Volume" as CinderType,
@VolumeSeconds:=SUM(IF(v.created_at <= starttime /* Captures Volumes which were created outside of the period deleted out of the period */
AND (v.deleted_at >= endtime
OR ISNULL(v.deleted_at)),
TIMESTAMPDIFF(SECOND,
starttime,
endtime),
IF(v.created_at <= starttime /* Captures Volumes which were created before the period and deleted during the period */
AND v.deleted_at < endtime,
TIMESTAMPDIFF(SECOND,
starttime,
v.deleted_at),
IF(v.created_at > starttime /* Captures Volumes which were created during the period and deleted outside the period */
AND (v.deleted_at >= endtime
OR ISNULL(v.deleted_at)),
TIMESTAMPDIFF(SECOND,
v.created_at,
endtime),
TIMESTAMPDIFF(SECOND,
v.created_at,
v.deleted_at))))) AS Volume_Seconds, /* Generates a count of seconds Volumes were running */
v.size AS Volume_GB
FROM
cinder.volumes v
JOIN
keystone.project p ON v.project_id = p.id
JOIN
keystone.project pp ON p.parent_id = pp.id
WHERE
v.created_at <= endtime
AND (v.deleted_at >= starttime
OR ISNULL(v.deleted_at))
GROUP BY v.availability_zone , v.size , p.name , pp.name;
END
48 changes: 48 additions & 0 deletions OpenStack-accounting/sql/glance_get_accounting_data.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime)
BEGIN
/*
This procedure generates accounting data for glance
*/
SELECT
p.name AS Project,
pp.name AS Department,
COUNT(g.id) AS Images,
ip.value as GlanceType,
if(il.value like "%rbd%", "RBD" ,if(il.value like "%swift%","OBJECT","UNKNONWN")) as StorageBackend,
@ImageSeconds:=SUM(IF(g.created_at <= starttime /* Captures Images which were created outside of the period deleted out of the period */
AND (g.deleted_at >= endtime
OR ISNULL(g.deleted_at)),
TIMESTAMPDIFF(SECOND,
starttime,
endtime),
IF(g.created_at <= starttime /* Captures Images which were created before the period and deleted during the period */
AND g.deleted_at < endtime,
TIMESTAMPDIFF(SECOND,
starttime,
g.deleted_at),
IF(g.created_at > starttime /* Captures Images which were created during the period and deleted outside the period */
AND (g.deleted_at >= endtime
OR ISNULL(g.deleted_at)),
TIMESTAMPDIFF(SECOND,
g.created_at,
endtime),
TIMESTAMPDIFF(SECOND,
g.created_at,
g.deleted_at))))) AS Image_Seconds, /* Generates a count of seconds Images were running */
g.size/(1024 * 1024 * 1024) AS Glance_GB
FROM
glance.images g
join
glance.image_properties ip on g.id = ip.image_id and ip.name = "image_type"
join
glance.image_locations il on g.id = il.image_id
JOIN
keystone.project p ON g.owner = p.id
JOIN
keystone.project pp ON p.parent_id = pp.id
WHERE
g.created_at <= endtime
AND (g.deleted_at >= starttime
OR g.deleted_at is null )
GROUP BY ip.value, g.size , p.name , pp.name,if(il.value like "%rbd%", "SIRIUS" ,if(il.value like "%swift%","ECHO","UNKNONWN"));
END
51 changes: 51 additions & 0 deletions OpenStack-accounting/sql/manila_get_accounting_data.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime)
BEGIN
/*
This procedure generates accounting data for manila
*/
SELECT
p.name AS Project,
pp.name AS Department,
maz.name AS Availability_zone,
st.name AS Share_type,
COUNT(m.id) AS Shares,
"Share" as ManilaType,
@ShareSeconds:=SUM(IF(m.created_at <= starttime /* Captures Shares which were created outside of the period deleted out of the period */
AND (m.deleted_at >= endtime
OR ISNULL(m.deleted_at)),
TIMESTAMPDIFF(SECOND,
starttime,
endtime),
IF(m.created_at <= starttime /* Captures Shares which were created before the period and deleted during the period */
AND m.deleted_at < endtime,
TIMESTAMPDIFF(SECOND,
starttime,
m.deleted_at),
IF(m.created_at > starttime /* Captures Shares which were created during the period and deleted outside the period */
AND (m.deleted_at >= endtime
OR ISNULL(m.deleted_at)),
TIMESTAMPDIFF(SECOND,
m.created_at,
endtime),
TIMESTAMPDIFF(SECOND,
m.created_at,
m.deleted_at))))) AS Share_Seconds, /* Generates a count of seconds Shares were running */
m.size AS Share_GB
FROM
manila.shares m
JOIN
manila.share_instances si ON m.id = si.share_id
JOIN
manila.share_types st ON si.share_type_id = st.id
JOIN
manila.availability_zones maz ON maz.id = si.availability_zone_id
JOIN
keystone.project p ON m.project_id = p.id
JOIN
keystone.project pp ON p.parent_id = pp.id
WHERE
m.created_at <= endtime
AND (m.deleted_at >= starttime
OR ISNULL(m.deleted_at))
GROUP BY m.size , p.name , pp.name , maz.name , st.name;
END
159 changes: 159 additions & 0 deletions OpenStack-accounting/sql/nova_get_accounting_data.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
CREATE PROCEDURE `get_accounting_data`(IN starttime datetime, IN endtime datetime)
BEGIN
/*
Generates accounting data from the Nova database on various resources used
Some further comments below for the trickier bits
*/
SELECT
IFNULL(i.availability_zone, 'nova') AS AvailabilityZone,
p.name AS Project,
pp.name AS Department,
it.name AS Flavor,
COUNT(i.uuid) as VMs,
@VMSeconds:=SUM(IF(i.created_at <= starttime /* Captures VMs which were created outside of the period deleted out of the period */
AND (i.deleted_at >= endtime
OR ISNULL(i.deleted_at)),
TIMESTAMPDIFF(SECOND,
starttime,
endtime),
IF(i.created_at <= starttime /* Captures VMs which were created before the period and deleted during the period */
AND i.deleted_at < endtime,
TIMESTAMPDIFF(SECOND,
starttime,
i.deleted_at),
IF(i.created_at > starttime /* Captures VMs which were created during the period and deleted outside the period */
AND (i.deleted_at >= endtime
OR ISNULL(i.deleted_at)),
TIMESTAMPDIFF(SECOND,
i.created_at,
endtime),
TIMESTAMPDIFF(SECOND,
i.created_at,
i.deleted_at))))) AS VM_Seconds, /* Generates a count of seconds VMs were running */
it.memory_mb AS Memory_MB,
it.vcpus AS VCPU,
it.swap AS Swap,
it.root_gb AS Root_GB,
it.ephemeral_gb AS Ephemeral_GB,
/* The below section extracts metadata to be used for accounting */
IFNULL((SELECT
value
FROM
nova_api.flavor_extra_specs es
WHERE
flavor_id = it.id
AND es.key LIKE '%per_unit_cost%'
AND es.key LIKE CONCAT('%',
CAST(YEAR(DATE_SUB(endtime, INTERVAL 3 MONTH))
AS NCHAR),
'%')
LIMIT 1),
0) AS Per_Unit_Cost,
IFNULL((SELECT
value
FROM
nova_api.flavor_extra_specs es
WHERE
flavor_id = it.id
AND es.key LIKE 'accounting:unit%'),
'core') AS Charge_Unit,
(SELECT
IFNULL(value, 0)
FROM
nova_api.flavor_extra_specs es
WHERE
flavor_id = it.id
AND es.key LIKE 'accounting:gpu_num%') AS GPU_Num
FROM
nova.instances i
JOIN
nova_api.flavors it ON i.instance_type_id = it.id
JOIN
keystone.project p ON i.project_id = p.id
JOIN
keystone.project pp ON p.parent_id = pp.id
WHERE
i.created_at <= endtime
AND (i.deleted_at >= starttime
OR ISNULL(i.deleted_at))
GROUP BY i.availability_zone , p.name , it.name , SUBSTRING(it.name,
1,
LOCATE('.', it.name))
UNION
SELECT
IFNULL(i.availability_zone, 'nova') AS AvailabilityZone,
p.name AS Project,
pp.name AS Department,
it.name AS Flavor,
COUNT(i.uuid) as VMs,
@VMSeconds:=SUM(IF(i.created_at <= starttime /* Captures VMs which were created outside of the period deleted out of the period */
AND (i.deleted_at >= endtime
OR ISNULL(i.deleted_at)),
TIMESTAMPDIFF(SECOND,
starttime,
endtime),
IF(i.created_at <= starttime /* Captures VMs which were created before the period and deleted during the period */
AND i.deleted_at < endtime,
TIMESTAMPDIFF(SECOND,
starttime,
i.deleted_at),
IF(i.created_at > starttime /* Captures VMs which were created during the period and deleted outside the period */
AND (i.deleted_at >= endtime
OR ISNULL(i.deleted_at)),
TIMESTAMPDIFF(SECOND,
i.created_at,
endtime),
TIMESTAMPDIFF(SECOND,
i.created_at,
i.deleted_at))))) AS VM_Seconds, /* Generates a count of seconds VMs were running */
it.memory_mb AS Memory_MB,
it.vcpus AS VCPU,
it.swap AS Swap,
it.root_gb AS Root_GB,
it.ephemeral_gb AS Ephemeral_GB,
/* The below section extracts metadata to be used for accounting */
IFNULL((SELECT
value
FROM
nova_api.flavor_extra_specs es
WHERE
flavor_id = it.id
AND es.key LIKE '%per_unit_cost%'
AND es.key LIKE CONCAT('%',
CAST(YEAR(DATE_SUB(endtime, INTERVAL 3 MONTH))
AS NCHAR),
'%')
LIMIT 1),
0) AS Per_Unit_Cost,
IFNULL((SELECT
value
FROM
nova_api.flavor_extra_specs es
WHERE
flavor_id = it.id
AND es.key LIKE 'accounting:unit%'),
'core') AS Charge_Unit,
(SELECT
IFNULL(value, 0)
FROM
nova_api.flavor_extra_specs es
WHERE
flavor_id = it.id
AND es.key LIKE 'accounting:gpu_num%') AS GPU_Num
FROM
nova.shadow_instances i
JOIN
nova_api.flavors it ON i.instance_type_id = it.id
JOIN
keystone.project p ON i.project_id = p.id
JOIN
keystone.project pp ON p.parent_id = pp.id
WHERE
i.created_at <= endtime
AND (i.deleted_at >= starttime
OR ISNULL(i.deleted_at))
GROUP BY i.availability_zone , p.name , it.name , SUBSTRING(it.name,
1,
LOCATE('.', it.name))
;
END
Loading

0 comments on commit 7708756

Please sign in to comment.