Commit 45940925 authored by David Seaward's avatar David Seaward

fixes from test attempt

* add rabbitmq and ssh/openvpn instructions to README
* correct celery worker/beat invocation
* add error_message to limitmonitor.external_credit (store the message when a credit attempt fails)
* warning but allow missing host key (ssh connection to openvpn server)
* syntax and other minor fixes
parent 0633ee58
......@@ -14,6 +14,9 @@ Prerequisites
* Python 3.4 or 3.5
* Django 1.11 (included in Python packages below)
* Nginx
* RabbitMQ server
* Accessible at `amqp://guest:guest@localhost:5672//`
* This can be achieved with just `apt install rabbitmq-server`
* Additional dependency packages:
* `libsasl2-dev`
* `libldap2-dev`
......@@ -28,7 +31,10 @@ Prerequisites
* External resources:
* LDAP database
* WooCommerce instance (REST API)
* RabbitMQ server
* SSH access to an OpenVPN server with `create_new_ovpn_config`
* The Nginx user (`www-data`) needs SSH access to the server
* Test with `sudo -u www-data ssh -p PORT REMOTE_USER@HOSTNAME`
* The user needing access can be changed in `purist_account_monitor.conf`
Other versions and alternatives may work but are untested.
......@@ -80,6 +86,7 @@ Setup
* `sudo apt install supervisor`
* `cp ./conf/supervisord/purist_account_monitor.conf /etc/supervisor/conf.d/`
* Restart services:
* `sudo service rabbitmq-server restart`
* `sudo service uwsgi-emperor restart`
* `sudo service nginx restart`
* `sudo service supervisor restart`
......
......@@ -9,7 +9,7 @@
; SPDX-License-Identifier: BSD-3-Clause
[program:purist_account_monitor]
command=/opt/purist/account_virtualenv/bin/celery beat --app purist_account --scheduler django_celery_beat.schedulers:DatabaseScheduler --loglevel=INFO
command=/opt/purist/account_virtualenv/bin/celery worker -B --app purist_account --scheduler django_celery_beat.schedulers:DatabaseScheduler --loglevel=INFO
directory=/opt/purist/account
user=www-data
numprocs=1
......
......@@ -16,7 +16,7 @@ class ExternalBundleAdmin(admin.ModelAdmin):
class ExternalCreditAdmin(admin.ModelAdmin):
list_display = ['parser', 'external_key', 'bundle_key', 'account_name', 'is_converted']
list_display = ['parser', 'external_key', 'bundle_key', 'account_name', 'is_converted', 'error_message']
class LimitAdmin(admin.ModelAdmin):
......
# -*- coding: utf-8 -*-
# Generated by Django 1.11.3 on 2017-08-15 11:59
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('limitmonitor', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='externalcredit',
name='error_message',
field=models.TextField(default=''),
),
migrations.AlterField(
model_name='limit',
name='is_active',
field=models.BooleanField(default=False),
),
migrations.AlterField(
model_name='limit',
name='time_total',
field=models.DecimalField(decimal_places=2, default=0, max_digits=6),
),
migrations.AlterField(
model_name='limit',
name='volume_total',
field=models.DecimalField(decimal_places=2, default=0, max_digits=6),
),
]
......@@ -103,6 +103,7 @@ class ExternalCredit(models.Model):
quantity = models.DecimalField(default=1, decimal_places=2, max_digits=6)
account_name = models.CharField(max_length=30, default="")
additional_data = models.TextField(default="")
error_message = models.TextField(default="")
is_converted = models.BooleanField(default=False)
created_date = models.DateTimeField(default=timezone.now)
updated_date = models.DateTimeField(default=timezone.now)
......
......@@ -19,15 +19,16 @@ def get_woo_connection():
consumer_key=settings.WOO_CONSUMER_KEY,
consumer_secret=settings.WOO_CONSUMER_SECRET,
wp_api=settings.WOO_WP_API,
version=settings.WOO_VERSION
version=settings.WOO_VERSION,
)
def get_openvpn_ssh_connection():
# make ssh connection to OpenVPN server
# (uses system host keys)
# (uses system host keys, warns if host is not recognised)
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.WarningPolicy()) # TODO: where is this logged?
ssh.connect(
hostname=settings.OVPN_HOSTNAME,
port=settings.OVPN_PORT,
......@@ -134,7 +135,7 @@ def update_limit_woo1(ssh, credit):
)
user = django.contrib.auth.get_user_model().objects.get(
username=username
username=username,
)
# set user email if missing
......@@ -180,6 +181,7 @@ def deactivate(ssh, limit):
@transaction.atomic
def store_credit_and_update_limit(ssh, credit):
try:
if credit.parser == "WOO1":
update_limit_woo1(ssh, credit)
......@@ -187,8 +189,11 @@ def store_credit_and_update_limit(ssh, credit):
raise Exception("Unrecognised parser " + credit.parser)
credit.is_converted = True
credit.error_message = ""
except Exception as e:
logger.exception("Skipped adding credit " + credit.parser + ":" + credit.external_key)
message = "Skipped adding credit " + credit.parser + ":" + credit.external_key + ". "
logger.exception(message)
credit.error_message = message + repr(e)
finally:
credit.save()
state = "converted" if credit.is_converted else "skipped"
......@@ -197,7 +202,7 @@ def store_credit_and_update_limit(ssh, credit):
@shared_task
def monitor_woo1():
# get API connection to WooCommerce
# make connection objects
woo_connection = get_woo_connection()
ssh = get_openvpn_ssh_connection()
......@@ -212,6 +217,7 @@ def monitor_woo1():
logger.exception("Skipping JSON entry " + str(json_entry))
# add new results
count = 0
for result in result_list:
try:
......@@ -224,29 +230,38 @@ def monitor_woo1():
quantity=result["quantity"],
account_name=result["account"],
additional_data=result["original_email"],
is_converted=False
is_converted=False,
)
if not is_existing_credit(credit):
store_credit_and_update_limit(ssh, credit)
count += 1
else:
logger.info("Skipped stored result " + str(result))
logger.debug("Skipped existing result " + str(result))
except Exception as e:
logger.exception("Skipping result " + str(result))
logger.exception("Skipped bad result " + str(result))
if count > 0:
logger.info("Added %i new results." % (count,))
@shared_task
def deactivate_all_expired_limits():
# make connection objects
ssh = get_openvpn_ssh_connection()
# get overdue objects and deactivate them
now = timezone.now()
overdue_list = Limit.objects.filter(expiry_date__lte=now, is_active=True)
ssh = get_openvpn_ssh_connection()
for limit in overdue_list:
deactivate(ssh, limit)
@shared_task
def debug_task(self):
def debug_connection_task():
# make connection objects
woo_connection = get_woo_connection()
logger.info("Debug task with " + str(woo_connection) + " completed successfully.")
ssh = get_openvpn_ssh_connection()
logger.info("Debug task with " + repr(woo_connection) + " and " + repr(ssh) + " completed successfully.")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment