/* dracal-sensgate-log: Connect to a SensGate via TCP/IP and log data from sensor devices.
 *
 * Copyright (C) 2020-2023  Dracal Technologies Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 3
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#include <stdio.h>
#include <stdlib.h>
#include <getopt.h>

#ifdef _WIN32
#include <io.h>
#define usleep(t) Sleep((t)/1000)
#define F_OK 0
#define access _access
#else
#include <unistd.h>
#endif

#include "tenkinet.h"
#include "datalog.h"
#include "csv.h"
#include "chip.h"
#include "source.h"
#include "timestamp.h"
#include "str_helper.h"
#include "heap.h"
#include "usbtenki_version.h"


#define VERSION_STRING "dracal-sensgate-log " USBTENKI_VERSION

#define INTERVAL_DEFAULT    TIMESTAMP_ONE_SECOND
#define INTERVAL_MIN        (TIMESTAMP_ONE_SECOND / 10)

static const char VERSION_TEXT[] =
    "\n"
    VERSION_STRING "\n"
    "Copyright (C) 2020-2025 Dracal Technologies Inc.\n"
    "\n"
    "This software is free software: you can redistribute it and/or modify it under the\n"
    "terms of the GNU General Public License as published by the Free Software Foundation,\n"
    "either version 3 of the License, or (at your option) any later version.\n"
    "\n"
    "This software is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n"
    "without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n"
    "See the GNU General Public License for more details.\n"
    "\n"
;

static const char HELP_TEXT[] =
    "\n"
    "Usage: dracal-sensgate-log [OPTION...] HOST[:PORT] [DEVICE[:CHANNELS]]...\n"
    "Connect to a SensGate via TCP/IP and log data from sensor devices.\n"
    "\n"
    "  -h, --help             Print this help screen\n"
    "  -v, --version          Print the version and copyright notice\n"
    "  -l, --list             Print a list of devices and channels available on the server\n"
    "  -L, --log=FILE         Log data to FILE in text format\n"
    "  -B, --binary-log=FILE  Log data to FILE in binary format (append to FILE if it already exists)\n"
    "  -r, --rows=N           Output N rows then stop\n"
    "  -m, --comment          Include a comment in the log file\n"
    "  -I, --interval=N       Set the logging interval in milliseconds (default: 1000)\n"
    "  -q, --quiet            Don't print any data to standard output (but still print errors)\n"
    "\n"
    "Virtual channel options:\n"
    "\n"
    "  -S, --slp=N     Set reference sea level pressure (kPa) for altitude (default: 101.325)\n"
    "  -o, --option=X  Enable special option (see below). You may specify -o multiple times.\n"
    "\n"
    "  Special options to specify with -o\n"
    "      no_humidex_range     Calculate humidex even if input values are out of range\n"
    "      no_heat_index_range  Calculate heat index even if input values are out of range\n"
    "\n"
    "Formatting options for text data:\n"
    "\n"
    "  -T, --temperature=UNIT    Temperature unit. Default: C\n"
    "  -P, --pressure=UNIT       Pressure unit. Default: kPa\n"
    "  -F, --frequency=UNIT      Frequency unit. Default: Hz\n"
    "  -M, --length=UNIT         Temperature unit. Default: m\n"
    "  -C, --concentration=UNIT  Concentration unit. Default: sensor default\n"
    "  -7, --ascii               Only print 7-bit ASCII characters (no Unicode symbols)\n"
    "  -x, --decimals=N          Number of fractional decimal digits to display. Default: automatic\n"
    "  -c, --decimal-comma       Use a comma as the decimal separator. Default: use a period\n"
    "  -s, --separator=STRING    Field separator string to use. Default: \"; \" (semicolon and space)\n"
    "  -e, --error=STRING        Error field value, or \"previous\" to repeat previous. Default: \"error\"\n"
    "  -t, --time=N              Time format; see explanation below\n"
    "\n"
    "The default PORT, if none is specified, is " STR(TENKINET_SERVER_PORT) ".\n"
    "\n"
    "When listing channels (-l option), all available devices/channels are listed.\n"
    "The program then exits immediately without logging any data.\n"
    "\n"
    "When logging to a new file with -L or -B, only the specified devices/channels are included.\n"
    "If none are specified, all available devices/channels are included.\n"
    "\n"
    "When appending to an existing binary file with -B, only the devices/channels already present in the\n"
    "file are included. Any devices/channels specified on the command-line are ignored, as well as -I\n"
    "and -c options, since the interval and comment are already defined in the existing log file.\n"
    "\n"
    "Binary logs are smaller than text logs, but they cannot be readily viewed. They can be inspected\n"
    "and converted to text using the dracal-sensgate-logtool utility.\n"
    "\n"
    "It is not possible to append to an existing text file. The -L option always creates a new file.\n"
    "\n"
    "Temperature units:\n"
    "    Celsius, C, Fahrenheit, F, Kelvin, K\n"
    "\n"
    "Pressure units:\n"
    "    kPa, hPa, Pa, bar, at (98.0665 kPa), atm (101.325 kPa), Torr, psi, inHg\n"
    "\n"
    "Frequency units:\n"
    "    mHz, Hz, kHz, MHz, rpm\n"
    "\n"
    "Length units:\n"
    "    mm, cm, dm, m, mil, in, ft, yd\n"
    "\n"
    "Concentration units:\n"
    "    ppb, ppm, percent\n"
    "\n"
    "Time formats:\n"
    "    0: None (don't print time)\n"
    "    1: Milliseconds since Epoch\n"
    "  * 2: System default (second-precision) * default\n"
    "    3: ISO 8601: Time only: HH:MM:SS\n"
    "    4: ISO 8601: Time only: HH:MM:SS.SSS\n"
    "    5: ISO 8601: Date and time in one field:  YYYY-mm-ddTHH:MM:SS\n"
    "    6: ISO 8601: Date and time in one field:  YYYY-mm-ddTHH:MM:SS.SSS\n"
    "    7: ISO 8601: Date and time in two fields: YYYY-mm-dd ; HH:MM:SS\n"
    "    8: ISO 8601: Date and time in two fields: YYYY-mm-dd ; HH:MM:SS.SSS\n"
    "\n"
    "Examples:\n"
    "\n"
    "  List all devices and channels on the SensGate at address 192.168.40.1 :\n"
    "    dracal-sensgate-log -l 192.168.40.1\n"
    "\n"
    "  Log all channels from 192.168.40.1 to standard output at the default interval:\n"
    "    dracal-sensgate-log 192.168.40.1\n"
    "\n"
    "  Like the previous example, but also log to a file:\n"
    "    dracal-sensgate-log -L MyLog.csv 192.168.40.1    # text format\n"
    "    dracal-sensgate-log -B MyLog.bin 192.168.40.1    # binary format\n"
    "\n"
    "  Log the first two channels of device E12345 and all channels of device E98765:\n"
    "    dracal-sensgate-log 192.168.40.1 E12345:0,1 E98765\n"
    "\n"
    "  Like the previous example, but logging to a text file and setting a one-minute log interval:\n"
    "    dracal-sensgate-log -L logfile.csv -I 60000 192.168.40.1 E12345:0,1 E98765\n"
    "\n"
;

static const struct option OPTIONS[] = {

    { "help", no_argument, NULL, 'h' },
    { "version", no_argument, NULL, 'v' },
    { "list", no_argument, NULL, 'l' },
    { "log", required_argument, NULL, 'L' },
    { "binary-log", required_argument, NULL, 'B' },
    { "rows", required_argument, NULL, 'r' },
    { "comment", required_argument, NULL, 'm' },
    { "interval", required_argument, NULL, 'I' },
    { "quiet", no_argument, NULL, 'q' },
    { "slp", required_argument, NULL, 'S' },
    { "option", required_argument, NULL, 'o' },
    { "temperature", required_argument, NULL, 'T' },
    { "pressure", required_argument, NULL, 'P' },
    { "frequency", required_argument, NULL, 'F' },
    { "length", required_argument, NULL, 'M' },
    { "concentration", required_argument, NULL, 'C' },
    { "ascii", no_argument, NULL, '7' },
    { "decimals", required_argument, NULL, 'x' },
    { "decimal-comma", no_argument, NULL, 'c' },
    { "separator", required_argument, NULL, 's' },
    { "error", required_argument, NULL, 'e' },
    { "time", required_argument, NULL, 't' },
    { NULL, 0, NULL, 0 }

};

typedef enum Mode {

    MODE_LOG = 0,
    MODE_LIST

} Mode;

typedef enum Error {  // TODO use error codes from tenkinet module instead
    SUCCESS = 0,
    ERROR_INVALID_OPTION,
    ERROR_MISSING_HOST,
    ERROR_INVALID_PORT,
    ERROR_EMPTY_SERIAL_NUMBER,
    ERROR_INVALID_CHANNEL_INDEX,
    ERROR_INIT,
    ERROR_CONNECTION,
    ERROR_SELECT,
    ERROR_DISCONNECTED,
    ERROR_DEVICE_NOT_FOUND,
    ERROR_MISSING_FILE_ARG,
    ERROR_TEXT_FILE_ALREADY_EXISTS,
    ERROR_CANNOT_OPEN_BINARY_FILE,
    ERROR_CANNOT_CREATE_BINARY_FILE,
    ERROR_CANNOT_CREATE_TEXT_FILE,
    ERROR_INVALID_UNIT,
    ERROR_INVALID_DECIMAL_DIGITS,
    ERROR_INVALID_TIME_FORMAT,
    DONE,  // not an error; indicates that we should disconnect from server and exit cleanly
} Error;

static const char* const ERRORS[] = {
    [SUCCESS] = "",
    [ERROR_INVALID_OPTION] = "Invalid option",
    [ERROR_MISSING_HOST] = "Missing HOST argument",
    [ERROR_INVALID_PORT] = "Invalid PORT argument",
    [ERROR_EMPTY_SERIAL_NUMBER] = "Empty DEVICE argument",
    [ERROR_INVALID_CHANNEL_INDEX] = "Invalid CHANNEL argument: must be a non-negative integer",
    [ERROR_INIT] = "Cannot initialize tenkinet module",
    [ERROR_CONNECTION] = "Could not connect to server",
    [ERROR_SELECT] = "Error in select()",
    [ERROR_DISCONNECTED] = "Disconnected from server",
    [ERROR_DEVICE_NOT_FOUND] = "Device not found",
    [ERROR_MISSING_FILE_ARG] = "Missing FILE argument",
    [ERROR_TEXT_FILE_ALREADY_EXISTS] = "Text FILE specified with -L already exists",
    [ERROR_CANNOT_OPEN_BINARY_FILE] = "Cannot open existing binary FILE specified with -B",
    [ERROR_CANNOT_CREATE_BINARY_FILE] = "Cannot create binary FILE specified with -B",
    [ERROR_CANNOT_CREATE_TEXT_FILE] = "Cannot create text FILE specified with -L",
    [ERROR_INVALID_UNIT] = "Invalid measurement unit",
    [ERROR_INVALID_DECIMAL_DIGITS] = "Invalid number of decimal digits",
    [ERROR_INVALID_TIME_FORMAT] = "Invalid time format",
};

typedef struct Subscription {

    Source *source;
    char serial_number[DEVICE_SERIAL_NUMBER_LEN + 1];
    List channel_indices;
    List channels;

} Subscription;


static Error error = SUCCESS;
static LIST(subscriptions);
static LIST(devices);
static LIST(offline_devices);

static DataLog *datalog;
static CSV csv_file;
static CSV csv_stdout;
static uint32_t line_counter;

static select_helper_data seldata;

// Configuration parsed from command-line arguments
static char *cfg_host = NULL;
static char opt_port[6] = STR(TENKINET_SERVER_PORT);
static Mode cfg_mode = MODE_LOG;
static uint32_t opt_interval = INTERVAL_DEFAULT;
static char *cfg_comment = "";
static char *cfg_csv_path = NULL;
static char *cfg_datalog_path = NULL;
static char cfg_quiet = 0;
static uint32_t opt_rows = -1;

static VirtualOptions cfg_virtual_options = VIRTUAL_OPTIONS_DEFAULT;

static CSV_Options cfg_csv_options = {

    .file = NULL,
    .datalog = NULL,

    .separator_str = "; ",
    .error_str = "error",
    .flags = 0,
    .frac_digits = -1,
    .time_format = CSV_TIME_SYSTEM_DEFAULT,

    .units = {
        [UNIT_CATEGORY_UNKNOWN] =  UNIT_SENSOR_DEFAULT,
        [UNIT_CATEGORY_TEMPERATURE] = UNIT_CELSIUS,
        [UNIT_CATEGORY_PRESSURE] = UNIT_KPA,
        [UNIT_CATEGORY_FREQUENCY] = UNIT_HZ,
        [UNIT_CATEGORY_VOLTAGE] =  UNIT_VOLT,
        [UNIT_CATEGORY_CURRENT] = UNIT_AMP,
        [UNIT_CATEGORY_POWER] = UNIT_WATT,
        [UNIT_CATEGORY_LENGTH] = UNIT_METER,
        [UNIT_CATEGORY_CONCENTRATION] = UNIT_SENSOR_DEFAULT,
        [UNIT_CATEGORY_RELATIVE_HUMIDITY] = UNIT_RH,
    }

};

static void quit(Error error);
static void parse_unit_argument(unit_category_t cat, const char *arg);
static void parse_host_and_port(char *arg);
static void parse_device_argument(char *arg);
static void open_datalog();
static Error create_datalog();

static int loop();

static Device *find_existing_device(const char *serial_number);

static void connection_callback(TenkinetClient client, int status, void *user_data);
static void list_callback(TenkinetClient client, void *user_data);
static void data_callback(TenkinetClient client, int64_t timestamp, void *user_data);
static void device_status_callback(TenkinetClient client, int64_t timestamp, Device *device, void *user_data);

static int device_list_compare(void *p, void *q);

static void print_list(TenkinetClient client);
static void print_device_info(Device *device);


static TenkinetCallbacks callbacks = {

    .connection_cb = connection_callback,
    .list_cb = list_callback,
    .data_cb = data_callback,
    .device_status_cb = device_status_callback,
    .user_data = NULL

};

int main(int argc, char *argv[]) {

    // If no argument is given, print help text and exit

    if (argc == 1) {
        fputs(HELP_TEXT, stdout);
        return 1;
    }

    // Parse command-line options

    int c;

    while ((c = getopt_long(argc, argv, "hvlL:B:r:m:I:qS:o:T:P:F:M:C:7x:cs:e:t:", OPTIONS, NULL)) != -1) {
        switch (c) {
            case 'h':
                fputs(HELP_TEXT, stdout);
                return 0;
            case 'v':
                fputs(VERSION_TEXT, stdout);
                return 0;
            case 'l':
                cfg_mode = MODE_LIST;
                break;
            case 'L':
                {
                    size_t len = strlen(optarg);
                    cfg_csv_path = malloc(len + 1);
                    memcpy(cfg_csv_path, optarg, len);
                    cfg_csv_path[len] = '\0';
                }
                break;
            case 'B':
                {
                    size_t len = strlen(optarg);
                    cfg_datalog_path = malloc(len + 1);
                    memcpy(cfg_datalog_path, optarg, len);
                    cfg_datalog_path[len] = '\0';
                }
                break;
            case 'r':
                opt_rows = strtoul(optarg, NULL, 10);
                break;
            case 'm':
                {
                    size_t len = strlen(optarg);
                    cfg_comment = malloc(len + 1);
                    memcpy(cfg_comment, optarg, len);
                    cfg_comment[len] = '\0';
                }
                break;
            case 'I':
                opt_interval = ((uint32_t)atoi(optarg)) * 1000;  // ms -> us
                if (opt_interval < INTERVAL_MIN) { opt_interval = INTERVAL_MIN; }
                break;
            case 'q':
                cfg_quiet = 1;
                break;
            case 'S':
				{
					char *e;
					double slp;

					slp = strtod(optarg, &e);
					if (e==optarg) {
						fprintf(stderr, "Invalid pressure\n");
						return -1;
					}
                    cfg_virtual_options.standard_sea_level_pressure = slp;
				}
                break;
            case 'o':
                if (!strcasecmp(optarg, "no_humidex_range")) {
                    cfg_virtual_options.flags |= VIRTUAL_FLAG_NO_HUMIDEX_RANGE;
                }
                else if (!strcasecmp(optarg, "no_heat_index_range")) {
                    cfg_virtual_options.flags |= VIRTUAL_FLAG_NO_HEAT_INDEX_RANGE;
                }
                else {
                    quit(ERROR_INVALID_OPTION);
                }
                break;
            case 'T':
                parse_unit_argument(UNIT_CATEGORY_TEMPERATURE, optarg);
                break;
            case 'F':
                parse_unit_argument(UNIT_CATEGORY_FREQUENCY, optarg);
                break;
            case 'P':
                parse_unit_argument(UNIT_CATEGORY_PRESSURE, optarg);
                break;
            case 'M':
                parse_unit_argument(UNIT_CATEGORY_LENGTH, optarg);
                break;
            case 'C':
                parse_unit_argument(UNIT_CATEGORY_CONCENTRATION, optarg);
                break;
            case '7':
                cfg_csv_options.flags |= CSV_FLAG_ASCII;
                break;
            case 'x':
                {
                    int a = atoi(optarg);
                    if (a < 0 || a > 9) {
                        quit(ERROR_INVALID_DECIMAL_DIGITS);
                    }
                    cfg_csv_options.frac_digits = a;
                }
                break;
            case 'c':
                cfg_csv_options.flags |= CSV_FLAG_DECIMAL_COMMA;
                break;
            case 's':
                cfg_csv_options.separator_str = optarg;
                break;
            case 'e':
                cfg_csv_options.error_str = strcasecmp(optarg, "previous") ? optarg : NULL;
                break;
            case 't':
                {
                    int a = atoi(optarg);
                    if (a < 0 || a > 8) {
                        quit(ERROR_INVALID_TIME_FORMAT);
                    }
                    cfg_csv_options.time_format = a;
                }
                break;
            default:
                quit(ERROR_INVALID_OPTION);
        }
    }

    // Next argument: HOST:PORT

    if (optind >= argc) {
        quit(ERROR_MISSING_HOST);
    }

    parse_host_and_port(argv[optind++]);

    if (cfg_mode == MODE_LOG) {

        // If a binary log file is specified, check if it already exists. If so, get
        // subscriptions from the file. Otherwise get subscriptions from the command-line.

        if (cfg_datalog_path && !access(cfg_datalog_path, F_OK)) {
            open_datalog();
        }
        else {
            // Get subs from command-line
            while (optind < argc) {
                char *arg = argv[optind++];
                parse_device_argument(arg);
            }
        }

        // Open text file for writing, if specified

        if (cfg_csv_path) {
            if (!access(cfg_csv_path, F_OK)) {
                // File exists
                quit(ERROR_TEXT_FILE_ALREADY_EXISTS);
            }
            cfg_csv_options.file = fopen(cfg_csv_path, "w");
            if (!cfg_csv_options.file) {
                quit(ERROR_CANNOT_CREATE_TEXT_FILE);
            }
            fprintf(stderr, "# Logging to new text file: %s\n", cfg_csv_path);
        }

    }

    // Client loop
    return loop();

}

static void quit(Error error) {

    if (error == DONE) {
        error = SUCCESS;
    }

    if (error) {
        fprintf(stderr, "# ERROR: %s\n", ERRORS[error]);
    }

    exit(error);

}

static void parse_unit_argument(unit_category_t cat, const char *arg) {

    unit_t unit = unit_parse(cat, arg);

    if (unit == UNIT_UNKNOWN) {
        quit(ERROR_INVALID_UNIT);
    }

    cfg_csv_options.units[cat] = unit;

}

static void parse_host_and_port(char *arg) {

    static const char COLON[] = ":";

    char *p = strtok(arg, COLON);
    if (!p) {
        quit(ERROR_MISSING_HOST);
    }

    size_t host_len = strlen(p);
    cfg_host = malloc(host_len + 1);
    memcpy(cfg_host, p, host_len);
    cfg_host[host_len] = 0;

    p = strtok(NULL, COLON);
    if (p) {
        memset(opt_port, 0, sizeof(opt_port));
        strncpy(opt_port, p, sizeof(opt_port) - 1);
    }

}

static void parse_device_argument(char *arg) {

    // Format
    //     DEVICE            all channels
    //     DEVICE:0,1,2,3    comma-separated list of channel indices

    // Find the end of the serial number in the arg string

    char *p;
    for (p = arg; *p != ':' && *p != 0 && (p - arg < DEVICE_SERIAL_NUMBER_LEN); p++);

    size_t serial_number_len = p - arg;

    if (serial_number_len == 0) {
        quit(ERROR_EMPTY_SERIAL_NUMBER);
    }

    // Find or create subscription

    Subscription *sub = NULL;

    LIST_FOR(&subscriptions) {
        Subscription *s = LIST_CUR(Subscription);
        if (!strncmp(s->serial_number, arg, serial_number_len)) {
            sub = s;
            break;
        }
    }

    if (sub == NULL) {
        sub = calloc(1, sizeof(Subscription));
        memcpy(sub->serial_number, arg, serial_number_len);  // null-terminated, thanks to calloc()
        list_add(&subscriptions, sub);
    }

    // Parse channel indices

    if (*p == 0) {  // no channel indices specified
        return;
    }

    static const char COMMA[] = ",";

    for (char *token = strtok(++p, COMMA); token != NULL; token = strtok(NULL, COMMA)) {

        // Validate token; simpler than error-checking with strtol()...
        for (p = token; *p != ',' && *p != 0; p++) {
            if (*p < '0' || '9' < *p) {
                quit(ERROR_INVALID_CHANNEL_INDEX);
            }
        }

        size_t channel_idx = atoi(token);
        list_add(&sub->channel_indices, (void *)channel_idx);

    }

    // Sort indices
    heap_sort(sub->channel_indices.elements, sub->channel_indices.size, compare_unsigned_integers);

}

static void open_datalog() {

    datalog = datalog_open_append(cfg_datalog_path);
    if (!datalog) {
        quit(ERROR_CANNOT_OPEN_BINARY_FILE);
    }

    fprintf(stderr, "# Appending to existing binary log file: %s\n", cfg_datalog_path);

    opt_interval = datalog->interval;

    list_grow(&subscriptions, datalog->devices.size);

    // Cursor to iterate over selected channels in order to determine their indices
    void **scp = datalog->channels.elements;
    void **scp_end = scp + datalog->channels.size;

    LIST_FOR(&datalog->devices) {

        Device *device = LIST_CUR(Device);

        Subscription *sub = calloc(1, sizeof(Subscription));
        list_add(&subscriptions, sub);

        memcpy(sub->serial_number, device->serial_number, DEVICE_SERIAL_NUMBER_LEN);
        list_grow(&sub->channel_indices, datalog->channels.size);

        size_t channel_idx = 0;

        LIST_FOR(&device->channels) {

            // Break if we're done iterating through selected channels
            if (scp >= scp_end) {
                break;
            }

            Channel *channel = LIST_CUR(Channel);
            Channel *sc = (Channel*)(*scp);

            if (sc == channel) {
                if (channel->chip_id >= USBTENKI_VIRTUAL_START) {
                    channel_idx = channel->chip_id;
                }
                list_add(&sub->channel_indices, (void *)channel_idx);
                scp++;
            }

            channel_idx++;

        }

    }

}

static Error create_datalog() {

    // Don't create if it already exists
    if (datalog != NULL) {
        return SUCCESS;
    }

    datalog = malloc(sizeof(DataLog));
    datalog->creation_timestamp = timestamp_now();
    datalog->interval = opt_interval;
    datalog->comment = cfg_comment;
    list_init(&datalog->devices);
    list_init(&datalog->channels);

    // Add subscribed devices and channels to the datalog

    LIST_FOR(&subscriptions) {

        Subscription *sub = LIST_CUR(Subscription);
        Source *source = sub->source;
        Device *device = device_clone(source->device);
        list_add(&datalog->devices, device);

        // Clone virtual channels and add them directly to the datalog device

        List *channels = &device->channels;
        list_grow(channels, channels->size + source->virtual_channels.size);

        LIST_FOR(&source->virtual_channels) {
            VirtualChannel *vc = LIST_CUR(VirtualChannel);
            Channel *channel = malloc(sizeof(Channel));
            *channel = vc->channel;
            list_add(channels, channel);
        }

        // Add selected channels to the datalog

        LIST_FOR(&sub->channel_indices) {

            void *p = LIST_CUR(void);
            size_t c = (size_t)p;

            if (c < USBTENKI_VIRTUAL_START) {
                // actual channel
                Channel *channel = LIST_GET(channels, c, Channel);
                list_add(&datalog->channels, channel);
            }
            else {
                // virtual channel
                LIST_FOR_REVERSE(channels) {
                    Channel *channel = LIST_CUR(Channel);
                    if (channel->chip_id == c) {
                        list_add(&datalog->channels, channel);
                        break;
                    }
                    else if (channel->chip_id < USBTENKI_VIRTUAL_START) {
                        break;
                    }
                }
            }

        }

    }

    // Create the datalog file, if requested

    if (cfg_datalog_path) {
        if (datalog_create(cfg_datalog_path, datalog)) {
            return ERROR_CANNOT_CREATE_BINARY_FILE;
        }
        fprintf(stderr, "# Logging to new binary file: %s\n", cfg_datalog_path);
    }

    return SUCCESS;

}

static int loop() {

    if (tenkinet_init(&seldata)) {
        quit(ERROR_INIT);
    }

    TenkinetClient client = tenkinet_client_new(cfg_host, opt_port, &callbacks);
    if (!client) {
        quit(ERROR_CONNECTION);
    }

    if (tenkinet_connect(client)) {
        quit(ERROR_CONNECTION);
    }

    while (!error) {

        if (select_helper(&seldata, NULL) < 0) {
            error = ERROR_SELECT;
            break;
        }

        tenkinet_process();

    }

    tenkinet_exit();
    quit(error);

    return error;

}

static Device *find_existing_device(const char *serial_number) {

    Device *device = device_find(&devices, serial_number, 0);

    if (device) {
        // That was easy!
        return device;
    }

    // Device is offline, but it may be referenced in the header of an existing datalog.
    // In this case, we already know the device's channels, and we can proceed.

    if (datalog) {
        device = device_find(&datalog->devices, serial_number, 0);
        if (device) {
            list_add(&offline_devices, device);
        }
    }

    return device;

}

static void connection_callback(TenkinetClient client, int status, void *user_data) {

    if (status == TENKINET_SUCCESS) {
        Version v = tenkinet_get_server_version(client);
        fprintf(stderr, "# Connected to %s, %s, v%u.%u.%u\n",
            tenkinet_get_server_name(client),
            tenkinet_get_server_serial_number(client),
            v.major,
            v.minor,
            v.revision
        );
        tenkinet_req_list(client);
        tenkinet_send(client);
    }
    else {
        if (!error) {
            error = ERROR_DISCONNECTED;
        }
    }

}

static void list_callback(TenkinetClient client, void *user_data) {

    list_remove_all(&devices);
    unsigned int n = tenkinet_get_devices(client, &devices);

    fprintf(stderr, "# Found %u device%s\n", n, (n > 1 ? "s" : ""));

    if (n == 0) {
        error = DONE;
        return;
    }

    if (cfg_mode == MODE_LIST) {
        print_list(client);
        error = DONE;
        return;
    }

    // If no subscriptions were defined, the default is to subscribe to everything

    if (subscriptions.size == 0) {

        list_grow(&subscriptions, n);

        LIST_FOR(&devices) {
            Device *device = LIST_CUR(Device);
            Subscription *sub = calloc(1, sizeof(Subscription));
            sub->source = source_new(device, &cfg_virtual_options);
            memcpy(sub->serial_number, device->serial_number, DEVICE_SERIAL_NUMBER_LEN);
            list_add(&subscriptions, sub);
        }

    }

    // Resolve subscribed devices and channels

    LIST_FOR(&subscriptions) {

        Subscription *sub = LIST_CUR(Subscription);
        Source *source = sub->source;
        Device *device = NULL;

        if (source) {
            device = source->device;
        }
        else {
            device = find_existing_device(sub->serial_number);
            if (!device) {
                fprintf(stderr, "# ERROR: Device not found: %s\n", sub->serial_number);
                error = ERROR_DEVICE_NOT_FOUND;
                return;
            }
            source = source_new(device, &cfg_virtual_options);
            sub->source = source;
        }

        if (sub->channel_indices.size > 0) {

            // Resolve channels from indices

            list_grow(&sub->channels, sub->channel_indices.size);

            LIST_FOR(&sub->channel_indices) {

                void *p = LIST_CUR(void);
                size_t c = (size_t)p;

                if (c < USBTENKI_VIRTUAL_START) {
                    // actual channel
                    list_add(&sub->channels, list_get(&source->calibrated_channels, c));
                }
                else {
                    // virtual channel
                    LIST_FOR(&source->virtual_channels) {
                        VirtualChannel *vc = LIST_CUR(VirtualChannel);
                        Channel *channel = &vc->channel;
                        if (channel->chip_id == c) {
                            list_add(&sub->channels, channel);
                            break;
                        }
                    }
                }

            }

        }
        else {

            // Subscribe to all channels (including virtual)

            size_t size = device->channels.size + source->virtual_channels.size;
            list_grow(&sub->channel_indices, size);
            list_grow(&sub->channels, size);

            for (size_t c = 0; c < device->channels.size; c++) {
                list_add(&sub->channel_indices, (void *)c);
                list_add(&sub->channels, list_get(&source->calibrated_channels, c));
            }

            LIST_FOR(&source->virtual_channels) {
                VirtualChannel *vc = LIST_CUR(VirtualChannel);
                size_t c = vc->channel.chip_id;
                list_add(&sub->channel_indices, (void *)c);
                list_add(&sub->channels, &vc->channel);
            }

        }

        tenkinet_req_subscribe(client, device);
        tenkinet_send(client);

    }

    error = create_datalog();
    if (error) {
        return;
    }

    cfg_csv_options.datalog = datalog;

    FILE *f = cfg_csv_options.file;
    if (f) {
        csv_file = csv_init(&cfg_csv_options);
        csv_write_header(csv_file);
    }

    if (!cfg_quiet) {
        fputc('\n', stderr);
        cfg_csv_options.file = stdout;
        csv_stdout = csv_init(&cfg_csv_options);
        cfg_csv_options.file = f;
        csv_write_header(csv_stdout);
    }

    tenkinet_req_poll(client, opt_interval);
    tenkinet_send(client);

}

static void data_callback(TenkinetClient client, int64_t timestamp, void *user_data) {

    // Refresh sources

    LIST_FOR(&subscriptions) {
        Subscription *sub = LIST_CUR(Subscription);
        Source *source = sub->source;
        source_refresh(source);
    }

    // Copy latest values to the datalog

    void **scp = datalog->channels.elements;

    LIST_FOR(&subscriptions) {
        Subscription *sub = LIST_CUR(Subscription);
        LIST_FOR(&sub->channels) {
            Channel *sub_channel = LIST_CUR(Channel);
            Channel *datalog_channel = *scp++;
            *datalog_channel = *sub_channel;
        }
    }

    datalog->timestamp = timestamp;

    // Write row

    if (cfg_datalog_path) {
        datalog_append_row(datalog);
    }

    if (csv_file) {
        csv_write_row(csv_file);
        fflush(cfg_csv_options.file);
    }

    if (csv_stdout) {
        csv_write_row(csv_stdout);
    }

    if (++line_counter >= opt_rows) {
        error = DONE;
    }

}

static void device_status_callback(TenkinetClient client, int64_t timestamp, Device *device, void *user_data) {

    // If device came online but wasn't previously listed, it means it was known from a datalog header.
    // In that case, update the corresponding source.

    if (device_flag_check(device, DEVICE_FLAG_ALIVE) && device_find(&offline_devices, device->serial_number, 1)) {
        LIST_FOR(&subscriptions) {
            Subscription *sub = LIST_CUR(Subscription);
            Source *source = sub->source;
            if (DEVICE_SERIAL_NUMBERS_EQUAL(source->device->serial_number, device->serial_number)) {
                source->device = device;
                // old source->device is still used by datalog, so don't delete!
                break;
            }
        }
    }

    // Print an informational message

    fprintf(stderr, "# Device #%u %s: %s, %s, v%u.%u\n",
        device->port,
        device_flag_check(device, DEVICE_FLAG_ALIVE) ? "connected" : "disconnected",
        device->product_name,
        device->serial_number,
        device->version.major,
        device->version.minor
    );

}

static int device_list_compare(void *p, void *q) {

    Device *a = (Device*)p;
    Device *b = (Device*)q;

    int c = ((int)(a->port)) - ((int)(b->port));

    if (c != 0) {
        return c;
    }

    return strncmp(a->serial_number, b->serial_number, DEVICE_SERIAL_NUMBER_LEN);

}

static void print_list(TenkinetClient client) {

    heap_sort(devices.elements, devices.size, device_list_compare);

    LIST_FOR(&devices) {
        Device *device = LIST_CUR(Device);
        print_device_info(device);
    }

}

static void print_device_info(Device *device) {

    Source source;

    source_init(&source, device, &cfg_virtual_options);

    printf("Device #%u: %s, %s, v%u.%u\n",
        device->port,
        device->product_name,
        device->serial_number,
        device->version.major,
        device->version.minor
    );

    for (int c = 0; c < device->channels.size; c++) {
        Channel *channel = LIST_GET(&device->channels, c, Channel);
        printf("\tChannel %u: %s [%s]\n",
            c,
            chip_description(channel->chip_id),
            chip_description_short(channel->chip_id)
        );
    }

    LIST_FOR(&source.virtual_channels) {
        VirtualChannel *vc = LIST_CUR(VirtualChannel);
        Channel *channel = &(vc->channel);
        printf("\tVirtual Channel %u: %s [%s]\n",
            channel->chip_id,
            chip_description(channel->chip_id),
            chip_description_short(channel->chip_id)
        );
    }

    source_clear(&source);

}
