raw.py

This is for raw, real time, loop data from a davis weather station. It’s been ever so slightly modified to persist data forever at a value of 0 via weewx.conf.

[Raw]
     data_binding = raw_binding
     data_limit = 0

The following code goes in place to /usr/share/weewx/user/raw.py

import syslog
import time
import weeutil.weeutil
import weewx
import weewx.engine
import weewx.manager
from weewx.engine import StdService

VERSION = "0.4.1-lh-bc"

#
# 2015-12-28 Modified by Luc Heijst to work with weewx version 3.3.1
# 2019-01-15 Minor tweak by Brett C to persist data... for a long time.
#

if weewx.__version__ < "3":
    raise weewx.UnsupportedFeature("weewx 3 is required, found %s" % weewx.__version__)

schema = [
    ('dateTime', 'INTEGER NOT NULL UNIQUE PRIMARY KEY'),
    ('usUnits', 'INTEGER NOT NULL'),
    #('interval', 'INTEGER NOT NULL DEFAULT \'1\''),
    ('barometer', 'REAL'),
    #('pressure', 'REAL'),
    #('altimeter', 'REAL'),
    ('inTemp', 'REAL'),
    ('outTemp', 'REAL'),
    ('inHumidity', 'REAL'),
    ('outHumidity', 'REAL'),
    ('windSpeed', 'REAL'),
    ('windDir', 'REAL'),
    #('windGust', 'REAL'),
    #('windGustDir', 'REAL'),
    ('rainRate', 'REAL'),
    ('rain', 'REAL'), # rain, was: dayRain
    ('dewpoint', 'REAL'),
    ('windchill', 'REAL'),
    ('heatindex', 'REAL'),
    #('rxCheckPercent', 'REAL'),
    ('dayET', 'REAL'), # ET
    ('radiation', 'REAL'),
    ('UV', 'REAL'),
    ('extraTemp1', 'REAL'),
    ('extraTemp2', 'REAL'),
    ('extraTemp3', 'REAL'),
    ('soilTemp1', 'REAL'),
    ('soilTemp2', 'REAL'),
    ('soilTemp3', 'REAL'),
    ('soilTemp4', 'REAL'),
    ('leafTemp1', 'REAL'),
    ('leafTemp2', 'REAL'),
    ('extraHumid1', 'REAL'),
    ('extraHumid2', 'REAL'),
    ('soilMoist1', 'REAL'),
    ('soilMoist2', 'REAL'),
    ('soilMoist3', 'REAL'),
    ('soilMoist4', 'REAL'),
    ('leafWet1', 'REAL'),
    ('leafWet2', 'REAL'),
    ('txBatteryStatus', 'REAL'),
    ('consBatteryVoltage', 'REAL')]
    #('hail', 'REAL'),
    #('hailRate', 'REAL'),
    #('heatingTemp', 'REAL'),
    #('heatingVoltage', 'REAL'),
    #('supplyVoltage', 'REAL'),
    #('referenceVoltage', 'REAL'),
    #('windBatteryStatus', 'REAL'),
    #('rainBatteryStatus', 'REAL'),
    #('outTempBatteryStatus', 'REAL'),
    #('inTempBatteryStatus', 'REAL')i

def get_default_binding_dict():
    return {'database': 'raw_mysql',
            'manager': 'weewx.manager.Manager',
            'table_name': 'raw',
            'schema': 'user.raw.schema'}

class RawService(StdService):

    def __init__(self, engine, config_dict):
        self.config_dict = config_dict
        super(RawService, self).__init__(engine, config_dict)
        syslog.syslog(syslog.LOG_INFO, "raw: service version is %s" % VERSION)

        d = config_dict.get('Raw', {})
        self.dataLimit = int(d.get('data_limit', 24))

        # get the database parameters we need to function
        self.binding = d.get('data_binding', 'raw_binding')
        self.dbm = self.engine.db_binder.get_manager(data_binding=self.binding, initialize=True)

        # be sure schema in database matches the schema we have
        dbcol = self.dbm.connection.columnsOf(self.dbm.table_name)
        dbm_dict = weewx.manager.get_manager_dict(
            config_dict['DataBindings'], config_dict['Databases'], self.binding)
        memcol = [x[0] for x in dbm_dict['schema']]
        if dbcol != memcol:
            raise Exception('raw: schema mismatch: %s != %s' % (dbcol, memcol))

        self.lastLoopDateTime = 0
        self.lastPrunedDateTime = 0
        self.bind(weewx.NEW_LOOP_PACKET, self.newLoopPacket)

    def prune_rawdata(self, dbm, ts, max_tries=3, retry_wait=10):
        """remove rawdata older than data_limit hours from the database"""
        sql = "delete from %s where dateTime < %d" % (dbm.table_name, ts)
        for count in range(max_tries):
            try:
                dbm.getSql(sql)
                syslog.syslog(syslog.LOG_INFO, 'raw: deleted rawdata prior to %s' % weeutil.weeutil.timestamp_to_string(ts))
                break
            except Exception, e:
                syslog.syslog(syslog.LOG_ERR, 'raw: prune failed (attempt %d of %d): %s' % ((count + 1), max_tries, e))
                syslog.syslog(syslog.LOG_INFO, 'raw: waiting %d seconds before retry' % retry_wait)
                time.sleep(retry_wait)
        else:
            raise Exception('raw: prune failed after %d attemps' % max_tries)

    def newLoopPacket(self, event):
        packet = event.packet
        prune_period = 300
        # It's possible for records with duplicate dateTimes - this occurs when an archive packet
        # is processed since the LOOP packets are queued up and then returned immediately when
        # looping resumes, coupled with the fact that for Vantage Pro consoles the dateTime value is
        # added by weewx. So, for database storage, skip the duplicates until we get a new one to
        # avoid a duplicate key error, but publish them all to redis regardless.
        dateTime = packet['dateTime']
        if dateTime != self.lastLoopDateTime:
            self.dbm.addRecord(packet)
            self.lastLoopDateTime = dateTime
        if dateTime > (self.lastPrunedDateTime + prune_period):
            if self.dataLimit is not 0:
                ts = ((dateTime - (self.dataLimit * 3600)) / prune_period) * prune_period  # preset on 5-min boundary
                self.prune_rawdata(self.dbm, ts, 2, 5)
            self.lastPrunedDateTime = dateTime
1 Comment
Oldest
Newest Most Voted
Inline Feedbacks
View all comments
trackback
28 December 2023 11:53 pm

[…] cmon, and neowx are no longer proactively used. As cmon had stability issues for me, and neowx was in use before Belchertown skin.NOTE: WeeWx loop data is also being dumped to the mySQL server in a second database. You’ll see it listed below as weewx_rapid. You can find details about WeeWx’s Raw script here. […]