Users disk usage

The following script is set to run every night via crontab

This get the usage of each user's home directory, puts it in a file. Then calls the python script to compare this to the previous day.

The scripts are here: /mnt/system_usage/

Data is in : /mnt/system_usage/data

> more users_disk_usage.sh
#!/bin/bash
#####################################################################
# script to calculate the usage of all users.
# report how much disc space is used.
# compare users usage to yesterday.
# Over time this can be plotted and we can see changes and spikes
# Author: Peter Thorpe 20181013
#####################################################################
d=$(date +%Y_%m_%d)
yest=$(date --date="yesterday" +"%Y_%m_%d")
echo "$d"

cd /mnt/system_usage

# STEP: 1
# get the disk space info
df_cmd="df > ./data/current_disks_status_${d}.txt"
echo ${df_cmd}
eval ${df_cmd}


# command to calc and sort the usage of all the users:
cd /storage/home/users

# STEP: 2
#  command to calc and sort the usage of all the users:
echo "calc: users usage."
# removed sort as this then requires du to have finished and
# takes too long  | sort -n -rn
du_cmd="du -s * >
           /mnt/system_usage/data/users_usage_${d}.txt"
echo ${du_cmd}
eval ${du_cmd}
wait


# STEP: 4
#  python convert bytes to human readable and compare with yesterday:
cd /mnt/system_usage
echo "chewing with python."
py_cmd="/mnt/apps/Python3.7/bin/python3.7 compare_usage.py
        --today /mnt/system_usage/data/users_usage_${d}.txt
        --yesterday /mnt/system_usage/data/users_usage_${yest}.txt
        -o /mnt/system_usage/data/USAGE_and_USAGE_CHANGE_${d}.txt"
echo ${py_cmd}
eval ${py_cmd}
wait




 > more compare_usage.py
#####!/mnt/apps/Python3.7/bin python3.7
#####!/usr/bin/env python # this one is 2.6
####################################################################
# title: Compare the users usage form yesterday to today
# Script take in the bytes usage of the users on a linux system.
# generated by: du -s * | sort -n -rn > users_usage_${d}.txt
# The date is specified by bash: d=$(date +%Y_%m_%d).
# Yesterday is specified by bash too:
# yest=$(date --date="yesterday" +"%Y_%m_%d")
# It parses the two files and compare the change in usage, and reports
# human readable usage values.
# Author: Peter Thorpe 20181013
###################################################################

import os
import sys
import errno
import logging
import logging.handlers
import sys
from collections import defaultdict
import argparse
import datetime
import time

if sys.version_info[:2] != (3, 6):
    # e.g. sys.version_info(major=3, minor=5, micro=2,
    # releaselevel='final', serial=0)
    # break the program
    if sys.version_info[:2] != (3, 7):
        print("currently using:", sys.version_info,
               "  version of python")
        raise ImportError("Python 3.6 or 3.7 is required for ")
        sys.exit(1)
VERSION = """compare usage v 0.001 \n"""
if "--version" in sys.argv:
    print(VERSION)
    sys.exit(1)


def get_args():
    parser = argparse.ArgumentParser(description="parse usage data " +
                                     "comper to yesterdays ",
                                     add_help=False)
    file_directory = os.path.realpath(__file__).split("compare_usage")[0]
    optional = parser.add_argument_group('optional arguments')

    optional.add_argument("-1", "--yesterday",
                          dest='yesterday',
                          action="store",
                          type=str,
                          default=None,
                          help="yesterday's usage file")

    optional.add_argument("-2", "--today",
                          dest='today',
                          action="store",
                          default=None,
                          type=str,
                          help="todays usage file")

    optional.add_argument("-o", "--out",
                          dest='out',
                          action="store",
                          default="usage_compared.out",
                          type=str,
                          help="todays usage file")

    optional.add_argument("--logfile",
                          dest="logfile",
                          action="store",
                          default="usage_calculation.log",
                          type=str,
                          help="Logfile name")
    args = parser.parse_args()
    return args, file_directory

##########################################
# Functions


def test_line(line):
    """returns true lines. Not comments or blank line"""
    if not line.strip():
        return False  # if the last line is blank
    if line.startswith("#"):
        return False  # comment line
    if line.startswith("    #"):
        return False  # comment line
    return line


def sizeof_fmt(num, suffix='B'):
    num = num * 1000
    for unit in [,'K','M','G','T','P','E','Z']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Yi', suffix)

def split_line(line):
    """func to split the output:
    in look like 59703192        /storage/home/users/pjt6/
    return usage and user and a list.
    """
    usage_bytes, user = line.split()
    return usage_bytes, user


def parse_file_with_dict(handle, dictionary):
    """func to populate the dic"""
    for line in handle:
        if test_line(line):
            usage_bytes, user = split_line(line)
            dictionary[user] = usage_bytes
    return dictionary

def get_usage_change(today, yesterday):
    """func takes in the bytes of today and yesterdays usage
    checks the biggest value and returns the change.
    Positive or negatve"""
    if today > yesterday:
        change = "+"
        usgae_change = today - yesterday
    else:
        usgae_change = yesterday - today
        change = "-"
    return usgae_change, change

FILECOUNT = 0
OLD_USAGE_DIC_COUNT = defaultdict(int)
NEW_USAGE_DIC_COUNT = defaultdict(int)
USAGE_CHANGE_DICT = defaultdict(str)

if __name__ == '__main__':
    # Set up logging
    args, FILE_DIRECTORY = get_args()
    cwd = os.getcwd()
    logger = logging.getLogger('compare_usage.py: %s' % time.asctime())
    logger.setLevel(logging.DEBUG)
    err_handler = logging.StreamHandler(sys.stderr)
    err_formatter = logging.Formatter('%(levelname)s: %(message)s')
    err_handler.setFormatter(err_formatter)
    logger.addHandler(err_handler)
    if args.logfile == "usage_calculation.log":
        DATE_TIME = "%s"  % (datetime.date.today())
        DATE_TIME = DATE_TIME.replace("-", "_")
        args.logfile = "%s_usage_calculation.log" % (DATE_TIME)
    try:
        logstream = open(args.logfile, 'w')
        err_handler_file = logging.StreamHandler(logstream)
        err_handler_file.setFormatter(err_formatter)
        # logfile is always verbose
        err_handler_file.setLevel(logging.INFO)
        logger.addHandler(err_handler_file)
    except:
        outstr = "Could not open %s for logging" % args.logfile
        logger.error(outstr)
        sys.exit(1)
    # Report input arguments
    logger.info(sys.version_info)
    logger.info("Command-line: %s", ' '.join(sys.argv))
    logger.info("Starting testing: %s", time.asctime())
    with open(args.yesterday) as handle:
        FILECOUNT += 1

        OLD_USAGE_DIC_COUNT = parse_file_with_dict(handle, OLD_USAGE_DIC_COUNT)
        # print(OLD_USAGE_DIC_COUNT)

    with open(args.today) as handle:
        FILECOUNT += 1
        NEW_USAGE_DIC_COUNT = parse_file_with_dict(handle, NEW_USAGE_DIC_COUNT)
    # outfile
    f_out = open(args.out, "w")
    f_out.write("user\tusage_today\tusage_yesterday\tusage_change\n")
    # compare the values in each dict for each user:
    # dictionary[user] = usage_bytes
    for user, usage_bytes in NEW_USAGE_DIC_COUNT.items():
        usage_bytes = int(usage_bytes)
        old_bytes = OLD_USAGE_DIC_COUNT[user]
        old_bytes = int(old_bytes)
        # fucntion should return a value then + or - depending on the usage change
        usgae_change, change = get_usage_change(usage_bytes, old_bytes)
        usgae_change_human = sizeof_fmt(usgae_change)
        USAGE_CHANGE_DICT[user] = "%s%s" % (change, usgae_change_human)
        outfmt = "%s\t%s\t%s\t%s\n" % (user, sizeof_fmt(usage_bytes),
                                       sizeof_fmt(old_bytes),
                                       usgae_change_human)
        f_out.write(outfmt)
    f_out.close()