Commit 1edf45fb authored by David Seaward's avatar David Seaward Committed by Gogs

Merge branch 'subscription_monitor_31' of david.seaward/purist_middleware into master

parents 1af2b871 2bbce29e
......@@ -29,3 +29,4 @@ OVPN_HOSTNAME = ssh.example.com
OVPN_PORT = 22
OVPN_USERNAME = username
OVPN_FILEPATH = "/path/to/{IDENTITY}/{IDENTITY}.ovpn"
SUBSCRIPTION_LINK = https://www.example.com
......@@ -8,20 +8,20 @@ from .models import Credit, ExternalBundle, ExternalCredit, Limit
#
class CreditAdmin(admin.ModelAdmin):
list_display = ['limit', 'external', 'time_credit', 'volume_credit']
list_display = ['id', 'limit', 'external', 'time_credit', 'volume_credit']
class ExternalBundleAdmin(admin.ModelAdmin):
list_display = ['parser', 'external_key', 'service', 'time_credit', 'volume_credit']
list_display = ['id', 'parser', 'external_key', 'service', 'time_credit', 'volume_credit']
class ExternalCreditAdmin(admin.ModelAdmin):
list_display = ['parser', 'external_key', 'bundle_key', 'account_name', 'is_converted', 'error_message']
list_display = ['label', 'external_code', 'bundle_key', 'account_name', 'is_converted', 'error_message']
class LimitAdmin(admin.ModelAdmin):
list_display = ['user', 'service', 'is_active', 'expiry_date', 'volume_total', 'time_total']
list_display = ['id', 'user', 'service', 'is_active', 'renewal_date', 'expiry_date', 'volume_total', 'time_total']
list_filter = ['service', 'renewal_date', 'user']
#
# Register admin models
......
......@@ -55,7 +55,7 @@ SPDX-License-Identifier: CC-BY-SA-4.0
<tr>
<th>{% trans "Service" %}</th>
<th>{% trans "Active" %}</th>
<th>{% trans "Expires" %}</th>
<th>{% trans "Renews/Expires" %}</th>
<th>{% trans "Credit" %}</th>
</tr>
</thead>
......@@ -65,8 +65,8 @@ SPDX-License-Identifier: CC-BY-SA-4.0
<tr>
<th style="text-align: left;">{{ limit.service_label() }}</th>
<td>{{ limit.is_active }}</td>
<td title="{{ limit.expiry_full_label() }}">{{ limit.expiry_short_label() }}</td>
<td>{{ limit.active_label() }}</td>
<td title="{{ limit.endpoint_full_label() }}">{{ limit.endpoint_short_label() }}</td>
<td>{{ limit.credit_label() }}</td>
</tr>
......@@ -79,14 +79,20 @@ SPDX-License-Identifier: CC-BY-SA-4.0
<ul>
<li><a href="{% url 'auth_password_change' %}">{% trans "Change password" %}</a></li>
<li><a href="{{ link_subscription }}" target="_blank">{% trans "Manage subscriptions" %}</a></li>
</ul>
<h2>{% trans "Downloads" %}</h2>
<ul>
{% if has_limit["TUNNEL"] %}
<li><a href="{% url 'ovpn_userfile' %}">{% trans "Tunnel certificate" %}</a></li>
<li><a href="{% static 'tunnel_instructions.pdf' %}">{% trans "How to use Tunnel certificate (PDF)" %}</a>
</li>
{% endif %}
{% if has_limit["NONE"] %}
<li>None</li>
{% endif %}
</ul>
</article>
......
# -*- coding: utf-8 -*-
# Generated by Django 1.11.4 on 2017-08-28 06:15
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('limitmonitor', '0002_auto_20170815_1159'),
]
operations = [
migrations.AddField(
model_name='limit',
name='renewal_date',
field=models.DateTimeField(blank=True, default=None, null=True),
),
migrations.AlterField(
model_name='externalbundle',
name='parser',
field=models.CharField(choices=[('WOO1', 'WooCommerce v1'), ('WOOSUB1', 'WooCommerce Subscription v1')], max_length=30),
),
migrations.AlterField(
model_name='externalcredit',
name='parser',
field=models.CharField(choices=[('WOO1', 'WooCommerce v1'), ('WOOSUB1', 'WooCommerce Subscription v1')], max_length=30),
),
]
......@@ -4,6 +4,7 @@ from django.utils import timezone
EXTERNAL_PARSER_CHOICES = (
("WOO1", "WooCommerce v1"),
("WOOSUB1", "WooCommerce Subscription v1"),
)
SERVICE_CHOICES = (
......@@ -21,6 +22,7 @@ def create_missing_user_limits(user):
class Limit(models.Model):
user = models.ForeignKey(settings.AUTH_USER_MODEL)
service = models.CharField(max_length=30, choices=SERVICE_CHOICES)
renewal_date = models.DateTimeField(default=None, blank=True, null=True)
expiry_date = models.DateTimeField(default=None, blank=True, null=True)
volume_total = models.DecimalField(default=0, decimal_places=2, max_digits=6)
time_total = models.DecimalField(default=0, decimal_places=2, max_digits=6)
......@@ -37,26 +39,42 @@ class Limit(models.Model):
return label
def expiry_short_label(self):
def active_label(self):
if self.expiry_date is None:
return "-"
if self.is_active:
return "Yes"
else:
return self.expiry_date.strftime("%Y-%m-%d")
return "No"
def endpoint_short_label(self):
return self.endpoint_label("%Y-%m-%d")
def endpoint_full_label(self):
return self.endpoint_label("%Y-%m-%d %H:%M:%S %z")
def expiry_full_label(self):
def endpoint_label(self, label_format):
if self.expiry_date is None:
is_blank = self.renewal_date is None and self.expiry_date is None
is_renewal = self.renewal_date is not None and self.expiry_date is None
is_expiry = self.renewal_date is None and self.expiry_date is not None
if is_blank:
return "-"
elif is_renewal:
return self.renewal_date.strftime(label_format)
elif is_expiry:
return self.expiry_date.strftime(label_format)
else:
return self.expiry_date.strftime("%Y-%m-%d %H:%M:%S %z")
return "Invalid"
def credit_label(self):
if self.service == "TUNNEL":
return self.tunnel_credit_label()
return self.days_credit_label()
else:
return self.tunnel_credit_label()
return self.days_credit_label()
def timedelta_to_daystring(self, delta):
......@@ -64,11 +82,12 @@ class Limit(models.Model):
days = int(round(delta.total_seconds() / single_day))
return str(days) + " days"
def tunnel_credit_label(self):
def days_credit_label(self):
has_zero = self.expiry_date is None and self.time_total == 0
has_credit_only = self.expiry_date is None and self.time_total > 0
has_valid_expiry = self.expiry_date is not None and self.time_total == 0
has_zero = self.expiry_date is None and self.renewal_date is None and self.time_total == 0
has_credit_only = self.expiry_date is None and self.renewal_date is None and self.time_total > 0
has_valid_expiry = self.expiry_date is not None and self.renewal_date is None and self.time_total == 0
has_valid_renewal = self.expiry_date is None and self.renewal_date is not None and self.time_total == 0
if has_zero:
label = "-"
......@@ -77,6 +96,9 @@ class Limit(models.Model):
elif has_valid_expiry:
difference = self.expiry_date - timezone.now()
label = self.timedelta_to_daystring(difference)
elif has_valid_renewal:
difference = self.renewal_date - timezone.now()
label = self.timedelta_to_daystring(difference)
else:
label = "Invalid"
......@@ -107,6 +129,14 @@ class ExternalCredit(models.Model):
created_date = models.DateTimeField(default=timezone.now)
updated_date = models.DateTimeField(default=timezone.now)
@property
def external_code(self):
return str(self.parser) + ":" + str(self.external_key)
@property
def external_bundle(self):
return str(self.parser) + ":" + str(self.external_key)
class Credit(models.Model):
limit = models.ForeignKey(Limit)
......
import paramiko
from celery.utils.log import get_task_logger
from django.conf import settings
from django.db import transaction
from django.utils import timezone
from woocommerce import API as WOO_API
import django.contrib.auth
from ..models import ExternalCredit, ExternalBundle, Limit
logger = get_task_logger(__name__)
def get_woo_connection():
return WOO_API(
url=settings.WOO_URL,
consumer_key=settings.WOO_CONSUMER_KEY,
consumer_secret=settings.WOO_CONSUMER_SECRET,
wp_api=settings.WOO_WP_API,
version=settings.WOO_VERSION,
)
def get_openvpn_ssh_connection():
# make ssh connection to OpenVPN server
# (uses system host keys, warns if host is not recognised)
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.WarningPolicy()) # TODO: where is this logged?
ssh.connect(
hostname=settings.OVPN_HOSTNAME,
port=settings.OVPN_PORT,
username=settings.OVPN_USERNAME,
)
return ssh
def managed_exec(ssh, command):
stdin, stdout, stderr = ssh.exec_command(command)
output = "".join(stdout.readlines()).strip()
if output == "":
output = "None."
error = "".join(stderr.readlines()).strip()
message = "Executed: %s Output: %s" % (command, output,)
# on sucess, log output, otherwise raise exception
if stdout.channel.recv_exit_status() == 0:
logger.info(message)
else:
message += " Error: " + error
raise Exception(message)
def is_existing_credit(credit):
matching_credits = ExternalCredit.objects.filter(
parser=credit.parser,
external_key=credit.external_key,
)
is_existing = len(matching_credits) > 0
return is_existing
def get_external_bundle(parser, external_key):
return ExternalBundle.objects.get(
parser=parser,
external_key=external_key,
)
def activate(ssh, limit, credit_timedelta=None, renewal_date=None):
is_credit = credit_timedelta is not None
is_renewal = renewal_date is not None
if is_credit == is_renewal:
raise Exception("Invalid activation attempt. Need strictly one of credit or renewal data.")
if is_credit and limit.is_active:
limit.expiry_date += credit_timedelta
elif is_credit and not limit.is_active:
limit.expiry_date = timezone.now() + credit_timedelta
limit.is_active = True
elif is_renewal:
limit.renewal_date = renewal_date
limit.is_active = True
else:
raise Exception("Invalid activation attempt. Unknown condition.")
# skip activation command if we are debugging
if settings.DEBUG_SKIP_ACTIVATION_COMMAND:
limit.save()
return
# otherwise, activate the limit before saving
if limit.service == "TUNNEL":
managed_exec(ssh, "./create_new_ovpn_config --generate %s" % (limit.user.get_identity(),))
limit.save()
def deactivate(ssh, limit):
if limit.service == "TUNNEL":
managed_exec(ssh, "./create_new_ovpn_config --revoke %s" % (limit.user.get_identity(),))
limit.is_active = False
limit.save()
def get_account_from_woo_meta(meta_list):
account = "invalid"
for meta_item in meta_list:
if meta_item["key"] in settings.WOO1_FIELD_LIST: # for example, "Existing username,"
account = meta_item["value"]
at_count = account.count("@")
if at_count == 0: # nodomain
account += "@" + settings.SITE_DOMAIN # corrected to nodomain@example.com
elif at_count == 1:
pass # valid account format
else: # at_count > 1, for example bad@user@example.com
account = account.replace("@", ".AT.") # force invalid name bad.AT.user.AT.example.com
return account
def get_limit_objects(credit):
# get and validate local username
suffix = "@" + settings.SITE_DOMAIN
if credit.account_name is None or not str.endswith(credit.account_name, suffix):
raise Exception("Invalid account name: " + str(credit.account_name))
else:
suffix_len = 0 - len(suffix)
username = credit.account_name[:suffix_len]
# get objects (implicit validation that they exist)
external_bundle = get_external_bundle(credit.parser, credit.bundle_key)
limit = Limit.objects.get(
user__username=username,
service=external_bundle.service,
)
user = django.contrib.auth.get_user_model().objects.get(
username=username,
)
return user, limit, external_bundle
@transaction.atomic
def store_credit_and_update_limit(ssh, credit, next_renewal=None):
try:
if credit.parser == "WOO1":
from .tunnel_credit import update_limit_woo1
update_limit_woo1(ssh, credit)
elif credit.parser == "WOOSUB1":
from .tunnel_subscription import update_limit_woosub1
update_limit_woosub1(ssh, credit, next_renewal)
else:
raise Exception("Unrecognised parser " + credit.parser)
credit.is_converted = True
credit.error_message = ""
except Exception as e:
message = "Skipped adding credit " + credit.parser + ":" + credit.external_key + ". "
logger.exception(message)
credit.error_message = message + repr(e)
finally:
credit.save()
state = "converted" if credit.is_converted else "skipped"
logger.info("Stored " + state + " credit " + credit.parser + ":" + credit.external_key)
def deactivate_all_expired_limits():
# make connection objects
ssh = get_openvpn_ssh_connection()
# get overdue objects and deactivate them
now = timezone.now()
overdue_list = Limit.objects.filter(expiry_date__lte=now, is_active=True)
for limit in overdue_list:
deactivate(ssh, limit)
def debug_connection_task():
# make connection objects
woo_connection = get_woo_connection()
ssh = get_openvpn_ssh_connection()
managed_exec(ssh, "whoami")
logger.info("Debug task with " + repr(woo_connection) + " and " + repr(ssh) + " completed successfully.")
from .common import *
def get_latest_woo1(connection, product_id):
return connection.get("orders?product=" + str(product_id))
def parse_woo1(json_entry, product_id):
result_list = []
order_name = json_entry["number"]
order_id = json_entry["id"]
for line_item in json_entry["line_items"]:
if line_item["product_id"] == product_id:
item_id = line_item["id"]
product_id = line_item["product_id"]
product_label = line_item["name"]
quantity = line_item["quantity"]
account = get_account_from_woo_meta(line_item["meta"])
external_key = str(order_id) + ":" + str(item_id)
external_label = str(order_name)
result_list.append({
"parser": "WOO1",
"external_key": external_key,
"label": external_label,
"product_key": product_id,
"product_label": product_label,
"quantity": quantity,
"account": account,
"original_email": "",
"isconverted": False,
})
return result_list
def update_limit_woo1(ssh, credit):
user, limit, external_bundle = get_limit_objects(credit)
# # set user email if missing
# credit_has_email = not (credit.additional_data is None or credit.additional_data == "")
# user_has_email = not (user.email is None or user.email == "")
#
# if user_has_email:
# pass
# elif credit_has_email:
# user.email = credit.additional_data
# user.save()
# else:
# logger.warn("No email address for credit " + credit.parser + " " + credit.external_key)
credit_days = int(external_bundle.time_credit * credit.quantity)
credit_timedelta = timezone.timedelta(days=credit_days)
activate(ssh, limit, credit_timedelta, None)
def monitor_woo1():
# make connection objects
woo_connection = get_woo_connection()
ssh = get_openvpn_ssh_connection()
# get product sales and parse the results
result_list = []
for product_id in settings.WOO_PRODUCT_LIST:
latest_woo1_json = get_latest_woo1(woo_connection, product_id).json()
for json_entry in latest_woo1_json:
try:
result_list.extend(parse_woo1(json_entry, product_id))
except Exception as e:
logger.exception("Skipping JSON entry " + str(json_entry))
# add new results
count = 0
for result in result_list:
try:
credit = ExternalCredit(
parser=result["parser"],
external_key=result["external_key"],
label=result["label"],
bundle_key=result["product_key"],
bundle_label=result["product_label"],
quantity=result["quantity"],
account_name=result["account"],
additional_data=result["original_email"],
is_converted=False,
)
if not is_existing_credit(credit):
store_credit_and_update_limit(ssh, credit)
count += 1
else:
logger.debug("Skipped existing result " + str(result))
except Exception as e:
logger.exception("Skipped bad result " + str(result))
if count > 0:
logger.info("Added %i new results." % (count,))
import datetime
from .common import *
def parse_woosub1(json_entry):
subscription_id = str(json_entry["id"])
# validation
if len(json_entry["line_items"]) != 1:
raise Exception("Too many line items in subscription %s" % (id, ))
line_item = json_entry["line_items"][0]
quantity = line_item["quantity"]
if quantity != 1:
raise Exception("Bad quantity %s in subscription %s" % (quantity, id, ))
# calculate next renewal date
next_renewal_naive = datetime.datetime.strptime(json_entry["next_payment_date"], "%Y-%m-%dT%H:%M:%S")
next_renewal = timezone.make_aware(next_renewal_naive)
# create result
result = {
"parser": "WOOSUB1",
"external_key": subscription_id,
"label": json_entry["number"],
"bundle_key": str(line_item["product_id"]),
"bundle_label": str(line_item["name"]),
"quantity": 1,
"account": get_account_from_woo_meta(line_item["meta"]),
"next_renewal": next_renewal,
}
return [result, ]
def monitor_woosub1_new_subscriptions():
# make connection objects
woo_connection = get_woo_connection()
ssh = get_openvpn_ssh_connection()
# initialise
result_list = []
latest_subscription_json = woo_connection.get("subscriptions?orderby=date&order=desc").json()
# parse recent subscriptions and store results
for json_entry in latest_subscription_json:
try:
result_list.extend(parse_woosub1(json_entry))
except Exception as e:
logger.exception("Skipping JSON entry " + str(json_entry))
# add new results
count = 0
for result in result_list:
try:
credit = ExternalCredit(
parser=result["parser"],
external_key=result["external_key"],
label=result["label"],
bundle_key=result["bundle_key"],
bundle_label=result["bundle_label"],
quantity=result["quantity"],
account_name=result["account"],
additional_data="None",
is_converted=False,
)
if not is_existing_credit(credit):
store_credit_and_update_limit(ssh, credit, result["next_renewal"])
count += 1
else:
logger.debug("Skipped existing result " + str(result))
except Exception as e:
logger.exception("Skipped bad result " + str(result))
def update_limit_woosub1(ssh, credit, renewal_date):
user, limit, external_bundle = get_limit_objects(credit)
activate(ssh, limit, None, renewal_date)
def monitor_woosub1_renewals():
# make connection objects
woo_connection = get_woo_connection()
ssh = get_openvpn_ssh_connection()
# get objects due for renewal (one hour grace / buffer)
now = timezone.now() - datetime.timedelta(hours=1)
overdue_list = Limit.objects.filter(renewal_date__lte=now,expiry_date=None, is_active=True)
count = 0
for limit in overdue_list:
identity = limit.user.get_identity()
woosub_bundle_list = ExternalBundle.objects.filter(service=limit.service, parser="WOOSUB1")
for bundle in woosub_bundle_list:
woosub_credit = ExternalCredit.objects.get(account_name=identity, bundle_key=bundle.external_key,
parser="WOOSUB1")
try:
subscription_query = "subscriptions/" + woosub_credit.external_key
woosub_json = woo_connection.get(subscription_query).json()
result = parse_woosub1(woosub_json)
# deactivate expired limits
if not result["active"]:
deactivate(ssh, limit)
woosub_credit.is_converted = False
woosub_credit.error_message = "Expired."
else:
store_credit_and_update_limit(ssh, woosub_credit, result["renewal"])
woosub_credit.is_converted = True
woosub_credit.error_message = ""
count += 1
except Exception as e:
message = "Skipping bad credit %s (%s)." % (woosub_credit.id, woosub_credit.external_code)
logger.exception(message)
woosub_credit.is_converted = False
woosub_credit.error_message = message + repr(e)
finally:
woosub_credit.save()
if count > 0:
logger.info("Updated %i subscriptions." % (count,))
import django.contrib.auth
import paramiko
from celery import shared_task
from celery.utils.log import get_task_logger
from django.conf import settings
from django.db import transaction
from django.utils import timezone
from woocommerce import API as WOO_API
from .models import ExternalCredit, ExternalBundle, Limit
logger = get_task_logger(__name__)
def get_woo_connection():
return WOO_API(
url=settings.WOO_URL,
consumer_key=settings.WOO_CONSUMER_KEY,
consumer_secret=settings.WOO_CONSUMER_SECRET,
wp_api=settings.WOO_WP_API,
version=settings.WOO_VERSION,
)
def get_openvpn_ssh_connection():
# make ssh connection to OpenVPN server
# (uses system host keys, warns if host is not recognised)
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.WarningPolicy()) # TODO: where is this logged?
ssh.connect(
hostname=settings.OVPN_HOSTNAME,
port=settings.OVPN_PORT,
username=settings.OVPN_USERNAME,
)
return ssh
def managed_exec(ssh, command):
stdin, stdout, stderr = ssh.exec_command(command)
output = "".join(stdout.readlines()).strip()
if output == "":
output = "None."
error = "".join(stderr.readlines()).strip()
message = "Executed: %s Output: %s" % (command, output,)
# on sucess, log output, otherwise raise exception
if stdout.channel.recv_exit_status() == 0:
logger.info(message)
else:
message += " Error: " + error
raise Exception(message)