Commit 5fe5b620 authored by David Seaward's avatar David Seaward

refactor existing limitmonitor tasks and add renewal field

parent 457c9600
......@@ -55,7 +55,7 @@ SPDX-License-Identifier: CC-BY-SA-4.0
<tr>
<th>{% trans "Service" %}</th>
<th>{% trans "Active" %}</th>
<th>{% trans "Expires" %}</th>
<th>{% trans "Renews/Expires" %}</th>
<th>{% trans "Credit" %}</th>
</tr>
</thead>
......@@ -66,7 +66,7 @@ SPDX-License-Identifier: CC-BY-SA-4.0
<tr>
<th style="text-align: left;">{{ limit.service_label() }}</th>
<td>{{ limit.is_active }}</td>
<td title="{{ limit.expiry_full_label() }}">{{ limit.expiry_short_label() }}</td>
<td title="{{ limit.endpoint_full_label() }}">{{ limit.endpoint_short_label() }}</td>
<td>{{ limit.credit_label() }}</td>
</tr>
......
......@@ -4,6 +4,7 @@ from django.utils import timezone
EXTERNAL_PARSER_CHOICES = (
("WOO1", "WooCommerce v1"),
("WOOSUB1", "WooCommerce Subscription v1"),
)
SERVICE_CHOICES = (
......@@ -21,6 +22,7 @@ def create_missing_user_limits(user):
class Limit(models.Model):
user = models.ForeignKey(settings.AUTH_USER_MODEL)
service = models.CharField(max_length=30, choices=SERVICE_CHOICES)
renewal_date = models.DateTimeField(default=None, blank=True, null=True)
expiry_date = models.DateTimeField(default=None, blank=True, null=True)
volume_total = models.DecimalField(default=0, decimal_places=2, max_digits=6)
time_total = models.DecimalField(default=0, decimal_places=2, max_digits=6)
......@@ -37,26 +39,35 @@ class Limit(models.Model):
return label
def expiry_short_label(self):
def endpoint_short_label(self):
if self.expiry_date is None:
return "-"
else:
return self.expiry_date.strftime("%Y-%m-%d")
return self.endpoint_label("%Y-%m-%d")
def endpoint_full_label(self):
def expiry_full_label(self):
return self.endpoint_label("%Y-%m-%d %H:%M:%S %z")
if self.expiry_date is None:
def endpoint_label(self, label_format):
is_blank = self.renewal_date is None and self.expiry_date is None
is_renewal = self.renewal_date is not None and self.expiry_date is None
is_expiry = self.renewal_date is None and self.expiry_date is not None
if is_blank:
return "-"
elif is_renewal:
return self.renewal_date.strftime(label_format)
elif is_expiry:
return self.expiry_date.strftime(label_format)
else:
return self.expiry_date.strftime("%Y-%m-%d %H:%M:%S %z")
return "Invalid"
def credit_label(self):
if self.service == "TUNNEL":
return self.tunnel_credit_label()
return self.days_credit_label()
else:
return self.tunnel_credit_label()
return self.days_credit_label()
def timedelta_to_daystring(self, delta):
......@@ -64,12 +75,12 @@ class Limit(models.Model):
days = int(delta.total_seconds() / single_day)
return str(days) + " days"
def tunnel_credit_label(self):
def days_credit_label(self):
label = "-"
has_zero = self.expiry_date is None and self.time_total == 0
has_credit_only = self.expiry_date is None and self.time_total > 0
has_valid_expiry = self.expiry_date is not None and self.time_total == 0
has_zero = self.expiry_date is None and self.renewal_date is None and self.time_total == 0
has_credit_only = self.expiry_date is None and self.renewal_date is None and self.time_total > 0
has_valid_expiry = self.expiry_date is not None and self.renewal_date is None and self.time_total == 0
has_valid_renewal = self.expiry_date is None and self.renewal_date is not None and self.time_total == 0
if has_zero:
label = "-"
......@@ -78,6 +89,9 @@ class Limit(models.Model):
elif has_valid_expiry:
difference = self.expiry_date - timezone.now()
label = self.timedelta_to_daystring(difference)
elif has_valid_renewal:
difference = self.renewal_date - timezone.now()
label = self.timedelta_to_daystring(difference)
else:
label = "Invalid"
......
import paramiko
from celery.utils.log import get_task_logger
from django.conf import settings
from django.db import transaction
from django.utils import timezone
from woocommerce import API as WOO_API
from ..models import ExternalCredit, ExternalBundle, Limit
logger = get_task_logger(__name__)
def get_woo_connection():
return WOO_API(
url=settings.WOO_URL,
consumer_key=settings.WOO_CONSUMER_KEY,
consumer_secret=settings.WOO_CONSUMER_SECRET,
wp_api=settings.WOO_WP_API,
version=settings.WOO_VERSION,
)
def get_openvpn_ssh_connection():
# make ssh connection to OpenVPN server
# (uses system host keys, warns if host is not recognised)
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.WarningPolicy()) # TODO: where is this logged?
ssh.connect(
hostname=settings.OVPN_HOSTNAME,
port=settings.OVPN_PORT,
username=settings.OVPN_USERNAME,
)
return ssh
def managed_exec(ssh, command):
stdin, stdout, stderr = ssh.exec_command(command)
output = "".join(stdout.readlines()).strip()
if output == "":
output = "None."
error = "".join(stderr.readlines()).strip()
message = "Executed: %s Output: %s" % (command, output,)
# on sucess, log output, otherwise raise exception
if stdout.channel.recv_exit_status() == 0:
logger.info(message)
else:
message += " Error: " + error
raise Exception(message)
def is_existing_credit(credit):
matching_credits = ExternalCredit.objects.filter(
parser=credit.parser,
external_key=credit.external_key,
)
is_existing = len(matching_credits) > 0
return is_existing
def get_external_bundle(parser, external_key):
return ExternalBundle.objects.get(
parser=parser,
external_key=external_key,
)
def activate(ssh, limit, credit_timedelta):
if limit.is_active:
limit.expiry_date += credit_timedelta
else:
limit.expiry_date = timezone.now() + credit_timedelta
limit.is_active = True
# skip activation command if we are debugging
if settings.DEBUG_SKIP_ACTIVATION_COMMAND:
limit.save()
return
if limit.service == "TUNNEL":
managed_exec(ssh, "./create_new_ovpn_config --generate %s" % (limit.user.get_identity(),))
limit.save()
def deactivate(ssh, limit):
if limit.service == "TUNNEL":
managed_exec(ssh, "./create_new_ovpn_config --revoke %s" % (limit.user.get_identity(),))
limit.is_active = False
limit.save()
@transaction.atomic
def store_credit_and_update_limit(ssh, credit):
try:
if credit.parser == "WOO1":
from .tunnel_credit import update_limit_woo1
update_limit_woo1(ssh, credit)
else:
raise Exception("Unrecognised parser " + credit.parser)
credit.is_converted = True
credit.error_message = ""
except Exception as e:
message = "Skipped adding credit " + credit.parser + ":" + credit.external_key + ". "
logger.exception(message)
credit.error_message = message + repr(e)
finally:
credit.save()
state = "converted" if credit.is_converted else "skipped"
logger.info("Stored " + state + " credit " + credit.parser + ":" + credit.external_key)
def deactivate_all_expired_limits():
# make connection objects
ssh = get_openvpn_ssh_connection()
# get overdue objects and deactivate them
now = timezone.now()
overdue_list = Limit.objects.filter(expiry_date__lte=now, is_active=True)
for limit in overdue_list:
deactivate(ssh, limit)
def debug_connection_task():
# make connection objects
woo_connection = get_woo_connection()
ssh = get_openvpn_ssh_connection()
managed_exec(ssh, "whoami")
logger.info("Debug task with " + repr(woo_connection) + " and " + repr(ssh) + " completed successfully.")
import django.contrib.auth
from .common import *
def get_latest_woo1(connection, product_id):
return connection.get("orders?product=" + str(product_id))
def parse_woo1(json_entry, product_id):
result_list = []
order_name = json_entry["number"]
order_id = json_entry["id"]
for line_item in json_entry["line_items"]:
if line_item["product_id"] == product_id:
item_id = line_item["id"]
product_id = line_item["product_id"]
product_label = line_item["name"]
quantity = line_item["quantity"]
account = "invalid"
for meta_item in line_item["meta"]:
if meta_item["key"] in settings.WOO1_FIELD_LIST: # for example, "Existing username,"
account = meta_item["value"]
at_count = account.count("@")
if at_count == 0: # nodomain
account += "@" + settings.SITE_DOMAIN # corrected to nodomain@example.com
elif at_count == 1:
pass # valid account format
else: # at_count > 1, for example bad@user@example.com
account = account.replace("@", ".AT.") # force invalid name bad.AT.user.AT.example.com
external_key = str(order_id) + ":" + str(item_id)
external_label = order_name + ":" + str(item_id)
result_list.append({
"parser": "WOO1",
"external_key": external_key,
"label": external_label,
"product_key": product_id,
"product_label": product_label,
"quantity": quantity,
"account": account,
"original_email": "",
"isconverted": False,
})
return result_list
def update_limit_woo1(ssh, credit):
# validate credit
suffix = "@" + settings.SITE_DOMAIN
if credit.account_name is None or not str.endswith(credit.account_name, suffix):
raise Exception("Invalid account name: " + str(credit.account_name))
else:
suffix_len = 0 - len(suffix)
username = credit.account_name[:suffix_len]
# get external references (implicit validation that they exist)
external_bundle = get_external_bundle(credit.parser, credit.bundle_key)
limit = Limit.objects.get(
user__username=username,
service=external_bundle.service,
)
user = django.contrib.auth.get_user_model().objects.get(
username=username,
)
# set user email if missing
credit_has_email = not (credit.additional_data is None or credit.additional_data == "")
user_has_email = not (user.email is None or user.email == "")
if user_has_email:
pass
elif credit_has_email:
user.email = credit.additional_data
user.save()
else:
logger.warn("No email address for credit " + credit.parser + " " + credit.external_key)
credit_days = int(external_bundle.time_credit * credit.quantity)
credit_timedelta = timezone.timedelta(days=credit_days)
activate(ssh, limit, credit_timedelta)
@transaction.atomic
def store_credit_and_update_limit(ssh, credit):
try:
if credit.parser == "WOO1":
update_limit_woo1(ssh, credit)
else:
raise Exception("Unrecognised parser " + credit.parser)
credit.is_converted = True
credit.error_message = ""
except Exception as e:
message = "Skipped adding credit " + credit.parser + ":" + credit.external_key + ". "
logger.exception(message)
credit.error_message = message + repr(e)
finally:
credit.save()
state = "converted" if credit.is_converted else "skipped"
logger.info("Stored " + state + " credit " + credit.parser + ":" + credit.external_key)
def monitor_woo1():
# make connection objects
woo_connection = get_woo_connection()
ssh = get_openvpn_ssh_connection()
# get product sales and parse the results
result_list = []
for product_id in settings.WOO_PRODUCT_LIST:
latest_woo1_json = get_latest_woo1(woo_connection, product_id).json()
for json_entry in latest_woo1_json:
try:
result_list.extend(parse_woo1(json_entry, product_id))
except Exception as e:
logger.exception("Skipping JSON entry " + str(json_entry))
# add new results
count = 0
for result in result_list:
try:
credit = ExternalCredit(
parser=result["parser"],
external_key=result["external_key"],
label=result["label"],
bundle_key=result["product_key"],
bundle_label=result["product_label"],
quantity=result["quantity"],
account_name=result["account"],
additional_data=result["original_email"],
is_converted=False,
)
if not is_existing_credit(credit):
store_credit_and_update_limit(ssh, credit)
count += 1
else:
logger.debug("Skipped existing result " + str(result))
except Exception as e:
logger.exception("Skipped bad result " + str(result))
if count > 0:
logger.info("Added %i new results." % (count,))
import django.contrib.auth
import paramiko
from celery import shared_task
from celery.utils.log import get_task_logger
from django.conf import settings
from django.db import transaction
from django.utils import timezone
from woocommerce import API as WOO_API
from .models import ExternalCredit, ExternalBundle, Limit
logger = get_task_logger(__name__)
def get_woo_connection():
return WOO_API(
url=settings.WOO_URL,
consumer_key=settings.WOO_CONSUMER_KEY,
consumer_secret=settings.WOO_CONSUMER_SECRET,
wp_api=settings.WOO_WP_API,
version=settings.WOO_VERSION,
)
def get_openvpn_ssh_connection():
# make ssh connection to OpenVPN server
# (uses system host keys, warns if host is not recognised)
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.WarningPolicy()) # TODO: where is this logged?
ssh.connect(
hostname=settings.OVPN_HOSTNAME,
port=settings.OVPN_PORT,
username=settings.OVPN_USERNAME,
)
return ssh
def managed_exec(ssh, command):
stdin, stdout, stderr = ssh.exec_command(command)
output = "".join(stdout.readlines()).strip()
if output == "":
output = "None."
error = "".join(stderr.readlines()).strip()
message = "Executed: %s Output: %s" % (command, output,)
# on sucess, log output, otherwise raise exception
if stdout.channel.recv_exit_status() == 0:
logger.info(message)
else:
message += " Error: " + error
raise Exception(message)
def get_latest_woo1(connection, product_id):
return connection.get("orders?product=" + str(product_id))
def parse_woo1(json_entry, product_id):
result_list = []
order_name = json_entry["number"]
order_id = json_entry["id"]
for line_item in json_entry["line_items"]:
if line_item["product_id"] == product_id:
item_id = line_item["id"]
product_id = line_item["product_id"]
product_label = line_item["name"]
quantity = line_item["quantity"]
account = "invalid"
for meta_item in line_item["meta"]:
if meta_item["key"] in settings.WOO1_FIELD_LIST: # for example, "Existing username,"
account = meta_item["value"]
if account.count("@") == 0: # nodomain
account += "@" + settings.SITE_DOMAIN # corrected to nodomain@example.com
elif account.count("@") > 1: # bad@user@example.com
raise Exception("Invalid username: " + account)
elif not account.endswith(settings.SITE_DOMAIN): # user@unrecognised.com
raise Exception("Bad username suffix: " + account)
external_key = str(order_id) + ":" + str(item_id)
external_label = order_name + ":" + str(item_id)
result_list.append({
"parser": "WOO1",
"external_key": external_key,
"label": external_label,
"product_key": product_id,
"product_label": product_label,
"quantity": quantity,
"account": account,
"original_email": "",
"isconverted": False,
})
return result_list
def is_existing_credit(credit):
matching_credits = ExternalCredit.objects.filter(
parser=credit.parser,
external_key=credit.external_key,
)
is_existing = len(matching_credits) > 0
return is_existing
def update_limit_woo1(ssh, credit):
# validate credit
suffix = "@" + settings.SITE_DOMAIN
if credit.account_name is None or not str.endswith(credit.account_name, suffix):
raise Exception("Invalid account name: " + str(credit.account_name))
else:
suffix_len = 0 - len(suffix)
username = credit.account_name[:suffix_len]
# get external references (implicit validation that they exist)
external_bundle = ExternalBundle.objects.get(
parser=credit.parser,
external_key=credit.bundle_key,
)
limit = Limit.objects.get(
user__username=username,
service=external_bundle.service,
)
user = django.contrib.auth.get_user_model().objects.get(
username=username,
)
# set user email if missing
credit_has_email = not (credit.additional_data is None or credit.additional_data == "")
user_has_email = not (user.email is None or user.email == "")
if user_has_email:
pass
elif credit_has_email:
user.email = credit.additional_data
user.save()
else:
logger.warn("No email address for credit " + credit.parser + " " + credit.external_key)
credit_days = int(external_bundle.time_credit * credit.quantity)
credit_timedelta = timezone.timedelta(days=credit_days)
activate(ssh, limit, credit_timedelta)
def activate(ssh, limit, credit_timedelta):
if limit.is_active:
limit.expiry_date += credit_timedelta
else:
limit.expiry_date = timezone.now() + credit_timedelta
limit.is_active = True
if limit.service == "TUNNEL":
managed_exec(ssh, "./create_new_ovpn_config --generate %s" % (limit.user.get_identity(),))
limit.save()
def deactivate(ssh, limit):
if limit.service == "TUNNEL":
managed_exec(ssh, "./create_new_ovpn_config --revoke %s" % (limit.user.get_identity(),))
limit.is_active = False
limit.save()
@transaction.atomic
def store_credit_and_update_limit(ssh, credit):
try:
if credit.parser == "WOO1":
update_limit_woo1(ssh, credit)
else:
raise Exception("Unrecognised parser " + credit.parser)
credit.is_converted = True
credit.error_message = ""
except Exception as e:
message = "Skipped adding credit " + credit.parser + ":" + credit.external_key + ". "
logger.exception(message)
credit.error_message = message + repr(e)
finally:
credit.save()
state = "converted" if credit.is_converted else "skipped"
logger.info("Stored " + state + " credit " + credit.parser + ":" + credit.external_key)
from .task_resources import common
from .task_resources import tunnel_credit
from .task_resources import tunnel_subscription
@shared_task
def monitor_woo1():
# make connection objects
woo_connection = get_woo_connection()
ssh = get_openvpn_ssh_connection()
# get product sales and parse the results
result_list = []
for product_id in settings.WOO_PRODUCT_LIST:
latest_woo1_json = get_latest_woo1(woo_connection, product_id).json()
for json_entry in latest_woo1_json:
try:
result_list.extend(parse_woo1(json_entry, product_id))
except Exception as e:
logger.exception("Skipping JSON entry " + str(json_entry))
def tunnel_refresh_subscription():
tunnel_subscription.do_nothing()
# add new results
count = 0
for result in result_list:
try:
credit = ExternalCredit(
parser=result["parser"],
external_key=result["external_key"],
label=result["label"],
bundle_key=result["product_key"],
bundle_label=result["product_label"],
quantity=result["quantity"],
account_name=result["account"],
additional_data=result["original_email"],
is_converted=False,
)
@shared_task
def tunnel_new_subscription():
pass
if not is_existing_credit(credit):
store_credit_and_update_limit(ssh, credit)
count += 1
else:
logger.debug("Skipped existing result " + str(result))
except Exception as e:
logger.exception("Skipped bad result " + str(result))
if count > 0:
logger.info("Added %i new results." % (count,))
@shared_task
def tunnel_new_credit():
tunnel_credit.monitor_woo1()
@shared_task
def deactivate_all_expired_limits():
# make connection objects
ssh = get_openvpn_ssh_connection()
# get overdue objects and deactivate them
now = timezone.now()
overdue_list = Limit.objects.filter(expiry_date__lte=now, is_active=True)
for limit in overdue_list:
deactivate(ssh, limit)
common.deactivate_all_expired_limits()
@shared_task
def debug_connection_task():
# make connection objects
woo_connection = get_woo_connection()
ssh = get_openvpn_ssh_connection()
managed_exec(ssh, "whoami")
logger.info("Debug task with " + repr(woo_connection) + " and " + repr(ssh) + " completed successfully.")
common.debug_connection_task()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment