Skip to content

save metrics data to csv file #10542

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
putcn merged 1 commit into from
May 10, 2018
Merged

save metrics data to csv file #10542

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions tools/aws_benchmarking/server/cluster_master.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import threading
import logging
import copy
import csv

import netaddr
import boto3
Expand Down Expand Up @@ -136,6 +137,12 @@ def str2bool(v):
parser.add_argument(
'--master_server_ip', type=str, default="", help="master server private ip")

parser.add_argument(
'--metric_data_identifier',
type=str,
default="**metrics_data: ",
help="key string to identify metrics data")

parser.add_argument(
'--no_clean_up',
type=str2bool,
Expand All @@ -155,6 +162,11 @@ def str2bool(v):

log_files = ["master.log"]

metrics = {}

metrics_csv_file_name = "metrics.csv"
is_metrics_file_created = False


def create_subnet():
# if no vpc id provided, list vpcs
Expand Down Expand Up @@ -329,12 +341,42 @@ def create_pservers():
cleanup(args.task_name)


def save_metrics_data(str_msg):
#parse msg
logging.info("found metrics data, saving it to csv file")
global is_metrics_file_created
metrics_raw = str_msg.split(",")
with open(args.log_path + metrics_csv_file_name, 'a') as csvfile:
csv_fieldnames = []
csv_write_data = {}
for metric in metrics_raw:
metric_data = metric.split("=")
metric_key = metric_data[0].strip()
metric_val = float(metric_data[1].strip())
if not metric_key in metrics:
metrics[metric_key] = []
metric_repo = metrics[metric_key]
metric_repo.append(metric_val)
csv_fieldnames.append(metric_key)
csv_write_data[metric_key] = metric_val
writer = csv.DictWriter(csvfile, fieldnames=csv_fieldnames)
if not is_metrics_file_created:
writer.writeheader()
is_metrics_file_created = True
writer.writerow(csv_write_data)
logging.info("csv file appended")


def log_to_file(source, filename):
if not filename in log_files:
log_files.append(filename)
with open(args.log_path + filename, "a") as log_file:
for line in iter(source.readline, ""):
log_file.write(line)
if (line.startswith(args.metric_data_identifier)):
#found key data, trying to add to csv
line = line.replace(args.metric_data_identifier, "")
save_metrics_data(line)


def parse_command(command_raw, defaults={}):
Expand Down