/*
    SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org>

    SPDX-License-Identifier: LGPL-2.0-or-later
*/

#include "gbfsjob.h"
#include "gbfsreader.h"
#include "geo/geojson_p.h"

#include <KPublicTransport/Location>

#include <QJsonObject>
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QNetworkReply>
#include <QPolygonF>
#include <QVersionNumber>

#include <cassert>
#include <cmath>

using namespace KPublicTransport;

GBFSJob::GBFSJob(QNetworkAccessManager *nam, QObject *parent)
    : QObject(parent)
    , m_nam(nam)
{
    assert(nam);
}

GBFSJob::~GBFSJob() = default;

GBFSJob::Error GBFSJob::error() const
{
    return m_error;
}

QString GBFSJob::errorMessage() const
{
    return m_errorMsg;
}

GBFSService GBFSJob::service() const
{
    return m_service;
}

void GBFSJob::setRequestedData(std::vector<GBFS::FileType> &&fileTypes)
{
    m_fileTypes = std::move(fileTypes);
}

void GBFSJob::discoverAndUpdate(const GBFSService &service)
{
    m_service = service;
    if (!m_service.systemId.isEmpty()) {
        m_store = GBFSStore(m_service.systemId);

        if (m_store.hasCurrentData(GBFS::Discovery)) {
            qDebug() << "reusing cached discovery data" << m_service.systemId;
            m_discoverDoc = m_store.loadData(GBFS::Discovery);
            parseDiscoverData();
            return;
        }
    }

    qDebug() << "fetching discovery data" << m_service.discoveryUrl;
    auto reply = m_nam->get(QNetworkRequest(m_service.discoveryUrl));
    connect(reply, &QNetworkReply::finished, this, [this, reply]() { discoverFinished(reply); });
}

void GBFSJob::discoverFinished(QNetworkReply *reply)
{
    reply->deleteLater();
    if (reply->error() != QNetworkReply::NoError) {
        if (m_previousDiscoveryUrl.isValid()) {
            qDebug() << "new version discovery failed, falling back to old one" <<  reply->errorString();
            m_service.discoveryUrl = m_previousDiscoveryUrl;
        } else {
            handleNetworkError(reply);
            return;
        }
    } else {
        m_discoverDoc = QJsonDocument::fromJson(reply->readAll());
    }
    parseDiscoverData();
}

void GBFSJob::parseDiscoverData()
{
    const auto top = m_discoverDoc.object();
    //qDebug() << QJsonDocument(top).toJson();

    const auto data = top.value(QLatin1String("data")).toObject();
    // pick the feeds with the best language for our current locale
    if (data.size() == 1) {
        // only one set of feeds
        m_feeds = data.begin().value().toObject().value(QLatin1String("feeds")).toArray();
        if (m_feeds.isEmpty()) { // invalid format for single feeds
            m_feeds = data.value(QLatin1String("feeds")).toArray();
        }
    } else if (!data.empty()) {
        const auto localeLangs = QLocale().uiLanguages();
        for (const auto &l : localeLangs) {
            m_feeds = data.value(l).toObject().value(QLatin1String("feeds")).toArray();
            if (m_feeds.isEmpty()) {
                m_feeds = data.value(l.toLower()).toObject().value(QLatin1String("feeds")).toArray();
            }
            if (m_feeds.empty() && l.size() > 2 && l[2] == QLatin1Char('-')) {
                m_feeds = data.value(l.left(2)).toObject().value(QLatin1String("feeds")).toArray();
            }
            if (!m_feeds.empty()) {
                break;
            }
        }
        // take the first one if we haven't found a better match
        if (m_feeds.empty()) {
            qDebug() << "picking first language, as none matches" << localeLangs;
            m_feeds = data.begin().value().toObject().value(QLatin1String("feeds")).toArray();
        }
    }
    if (m_feeds.empty()) {
        m_error = DataError;
        m_errorMsg = QStringLiteral("no feed found in discovery response!");
        Q_EMIT finished();
        return;
    }

    m_state = m_state == State::Discover ? State::Version : State::SystemInformation;
    processFeeds();
}

void GBFSJob::processFeeds()
{
    bool proccedAtLeastOneFeed = false;
    const auto state = m_state; // can change as result of processing
    for (const auto &feedVal : std::as_const(m_feeds)) {
        const auto feed = feedVal.toObject();
        const auto name = feed.value(QLatin1String("name")).toString();
        const auto type = GBFS::typeForKeyName(name);
        const auto url = QUrl(feed.value(QLatin1String("url")).toString());

        switch (type) {
            case GBFS::SystemInformation:
                if (state != State::SystemInformation) {
                    continue;
                }
                break;
            case GBFS::Versions:
                if (state != State::Version) {
                    continue;
                }
                break;
            case GBFS::StationInformation:
            case GBFS::StationStatus:
            case GBFS::FreeBikeStatus:
            case GBFS::VehicleTypes:
            case GBFS::GeofencingZones:
                if (state != State::Data || !shouldFetchFile(type)) {
                    continue;
                }
                break;
            case GBFS::Discovery:
            case GBFS::SystemHours:
            case GBFS::SystemCalendar:
            case GBFS::SystemRegions:
            case GBFS::SystemPricingPlans:
            case GBFS::SystemAlerts:
                continue;
            default:
                qDebug() << "Unhandled feed:" << name << url;
                continue;
        }

        if (!m_store.isValid() || !m_store.hasCurrentData(type)) {
            qDebug() << "fetching" << name;
            auto reply = m_nam->get(QNetworkRequest(url));
            connect(reply, &QNetworkReply::finished, this, [this, reply, type]() { fetchFinished(reply, type); });
            ++m_pendingJobs;
        } else {
            parseData(m_store.loadData(type), type);
        }
        proccedAtLeastOneFeed = true;
    }

    if (!proccedAtLeastOneFeed) {
        switch (m_state) {
            case State::Version:
                m_state = State::SystemInformation;
                break;
            case State::SystemInformation:
            case State::Data:
                m_error = DataError;
                m_errorMsg = m_state == State::SystemInformation ? QStringLiteral("no system information") : QStringLiteral("no data");
                Q_EMIT finished();
                return;
            default:
                Q_UNREACHABLE();
        }
        QMetaObject::invokeMethod(this, &GBFSJob::processFeeds, Qt::QueuedConnection);
    } else if (m_pendingJobs == 0 && state == State::Data) {
        finalize();
    }
}

void GBFSJob::fetchFinished(QNetworkReply *reply, GBFS::FileType type)
{
    reply->deleteLater();
    --m_pendingJobs;
    const auto state = m_state; // can change as part of processing

    if (reply->error() != QNetworkReply::NoError) {
        // don't consider geofencing_zones failure fatal
        if (type != GBFS::GeofencingZones) {
            handleNetworkError(reply);
            return;
        } else {
            qDebug() << reply->url() << reply->errorString();
        }
    } else {
        const auto doc = QJsonDocument::fromJson(reply->readAll());
        if (m_store.isValid()) {
            m_store.storeData(type, doc);
        }
        parseData(doc, type);
    }

    if (m_pendingJobs == 0 && state == State::Data) {
        finalize();
    }
}

void GBFSJob::handleNetworkError(QNetworkReply *reply)
{
    m_error = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == 429 ? TooManyRequestsError : NetworkError;
    m_errorMsg = reply->errorString();
    if (m_pendingJobs == 0) { // wait for the rest to finish otherwise, to avoid double finished() emission
        Q_EMIT finished();
    }
}

void GBFSJob::parseData(const QJsonDocument &doc, GBFS::FileType type)
{
    switch (type) {
        case GBFS::SystemInformation:
            parseSystemInformation(doc);
            break;
        case GBFS::StationInformation:
            parseStationInformation(doc);
            break;
        case GBFS::FreeBikeStatus:
            parseFreeBikeStatus(doc);
            break;
        case GBFS::Versions:
            parseVersionData(doc);
            break;
        case GBFS::GeofencingZones:
            parseGeofencingZones(doc);
        default:
            break;
    }
}

void GBFSJob::parseSystemInformation(const QJsonDocument &doc)
{
    const auto systemId = GBFSReader::dataValue(doc, QLatin1String("system_id")).toString();
    if (systemId.isEmpty()) {
        m_error = DataError;
        m_errorMsg = QStringLiteral("unable to determine system_id!");
        Q_EMIT finished();
        return;
    }
    if (m_service.systemId.isEmpty()) {
        m_service.systemId = systemId;
    }
    m_store = GBFSStore(m_service.systemId);
    m_store.storeData(GBFS::Discovery, m_discoverDoc);
    m_store.storeData(GBFS::SystemInformation, doc);
    if (!m_versionDoc.isEmpty()) {
        m_store.storeData(GBFS::Versions, m_versionDoc);
    }

    m_state = State::Data;
    QMetaObject::invokeMethod(this, &GBFSJob::processFeeds, Qt::QueuedConnection);
}

void GBFSJob::parseStationInformation(const QJsonDocument &doc)
{
    const auto stations = GBFSReader::dataValue(doc, QLatin1String("stations")).toArray();
    collectCoordinates(stations);
    qDebug() << stations.size() << "stations/docks";
}

void GBFSJob::parseFreeBikeStatus(const QJsonDocument &doc)
{
    const auto bikes = GBFSReader::dataValue(doc, QLatin1String("bikes")).toArray();
    collectCoordinates(bikes);
    qDebug() << bikes.size() << "free floating vehicles";
}

static void filterOutliers(const std::vector<double> &values, double &minVal, double &maxVal, const std::function<double(double, double)> &distFunc)
{
    // first step: primitive distance-based trimming at the extremes
    auto beginIt = values.begin();
    while(std::next(beginIt) != values.end()) {
        if (distFunc(*beginIt, *std::next(beginIt)) > 50'000) {
            ++beginIt;
        } else {
            break;
        }
    }
    auto endIt = std::prev(values.end());
    while(endIt != beginIt && std::prev(endIt) != beginIt) {
        if (distFunc(*endIt, *std::prev(endIt)) > 50'000) {
            --endIt;
        } else {
            break;
        }
    }
    ++endIt;

    // second step: standard deviation
    const auto n = std::distance(beginIt, endIt);
    const auto mean = std::accumulate(beginIt, endIt, 0.0, [n](auto a, auto b) { return a + b / n; });
    auto sigma = std::accumulate(beginIt, endIt, 0.0, [n](auto a, auto b) {
        return a + (std::pow(b, 2.0) / n);
    });
    sigma = std::sqrt(sigma - std::pow(mean, 2.0)) * 3.0;

    auto lowerBound = mean - sigma;
    auto it = std::lower_bound(values.begin(), values.end(), lowerBound);
    if (it != values.end()) {
        lowerBound = (*it);
    }
    auto upperBound = mean + sigma;
    it = std::lower_bound(values.begin(), values.end(), upperBound);
    if (it != values.begin()) {
        upperBound = *(std::prev(it));
    }

    minVal = std::min(minVal, std::max(lowerBound, values.front())); // clamp by 3 sigma, but don't exceed the input range when not needed
    maxVal = std::max(maxVal, std::min(upperBound, values.back()));
}

void GBFSJob::collectCoordinates(const QJsonArray &array)
{
    m_latitudes.reserve(m_latitudes.size() + array.size());
    m_longitudes.reserve(m_longitudes.size() + array.size());

    for (const auto &statVal : array) {
        const auto station = statVal.toObject();
        const auto lat = GBFSReader::readLatitude(station);
        if (!std::isnan(lat) && lat >= -90.0 && lat <= 90.0 && std::abs(lat) > 0.001) {
            m_latitudes.push_back(lat);
        }
        const auto lon = GBFSReader::readLongitude(station);
        if (!std::isnan(lon) && lon >= -180.0 && lon <= 180.0 && std::abs(lon) > 0.001) {
            m_longitudes.push_back(lon);
        }
    }
}

void GBFSJob::parseVersionData(const QJsonDocument &doc)
{
    m_versionDoc = doc;
    const auto versions = GBFSReader::dataValue(doc, QLatin1String("versions")).toArray();
    QJsonObject bestVersion;
    for (const auto &verVal : versions) {
        const auto version = verVal.toObject();
        if (bestVersion.isEmpty()) {
            bestVersion = version;
        }
        if (QVersionNumber::fromString(bestVersion.value(QLatin1String("version")).toString()) < QVersionNumber::fromString(version.value(QLatin1String("version")).toString())) {
            bestVersion = version;
        }
    }

    const auto url = QUrl(bestVersion.value(QLatin1String("url")).toString());
    if (!url.isEmpty() && m_service.discoveryUrl != url) {
        qDebug() << "found newer version:" << url << m_service.discoveryUrl;
        m_previousDiscoveryUrl = m_service.discoveryUrl;
        m_service.discoveryUrl = url;
        m_state = State::DiscoverRestart;
        discoverAndUpdate(m_service);
    } else {
        m_state = State::SystemInformation;
        QMetaObject::invokeMethod(this, &GBFSJob::processFeeds, Qt::QueuedConnection);
    }
}

void GBFSJob::parseGeofencingZones(const QJsonDocument &doc)
{
    const auto features = GBFSReader::dataValue(doc, QLatin1String("geofencing_zones")).toObject()
        .value(QLatin1String("features")).toArray();
    for (const auto &featureVal : features) {
        const auto geo = featureVal.toObject().value(QLatin1String("geometry")).toObject();
        const auto rect = GeoJson::readOuterPolygon(geo).boundingRect();
        if (rect.isNull() || rect.left() < -180.0 || rect.right() > 180.0 || rect.top() < -90.0 || rect.bottom() > 90.0) {
            qDebug() << "invalid geofence box:" << rect;
            continue;
        }
        // we need to run this through outlier filtering as well, we got random nonsense elements in a few cities as well
        m_latitudes.push_back(rect.top());
        m_latitudes.push_back(rect.bottom());
        m_longitudes.push_back(rect.left());
        m_longitudes.push_back(rect.right());
    }
}

void GBFSJob::finalize()
{
    // add a 500m radius for single points
    if (m_latitudes.size() == 1) {
        const auto d = 250.0 / Location::distance(m_latitudes.front(), 0.0, m_latitudes.front() + 1.0, 0.0);
        m_latitudes.push_back(m_latitudes.front() - d);
        m_latitudes.push_back(m_latitudes.front() + d);
    }
    if (m_longitudes.size() == 1 && !m_latitudes.empty()) {
        const auto d = 250.0 / Location::distance(m_latitudes.front(), m_longitudes.front(), m_latitudes.front(), m_longitudes.front() + 1.0);
        m_longitudes.push_back(m_longitudes.front() - d);
        m_longitudes.push_back(m_longitudes.front() + d);
    }

    double minLat = 90.0, maxLat = -90.0, minLon = 180.0, maxLon = -180.0;
    if (!m_latitudes.empty() && !m_longitudes.empty()) {
        std::sort(m_latitudes.begin(), m_latitudes.end());
        std::sort(m_longitudes.begin(), m_longitudes.end());

        // covered area is reasonable, take as-is
        if (Location::distance(m_latitudes.front(), m_longitudes.front(), m_latitudes.back(), m_longitudes.back()) <= 50'000) {
            minLat = m_latitudes.front();
            minLon = m_longitudes.front();
            maxLat = m_latitudes.back();
            maxLon = m_longitudes.back();
        } else {
            // try to filter out outliers
            filterOutliers(m_latitudes, minLat, maxLat, [](auto lat1, auto lat2) { return Location::distance(lat1, 0.0, lat2, 0.0); });
            filterOutliers(m_longitudes, minLon, maxLon, [&](auto lon1, auto lon2) {
                const auto lat = (maxLat - minLat) / 2.0;
                return Location::distance(lat, lon1, lat, lon2);
            });
        }
    }

    if (maxLat > minLat && maxLon > minLon) {
        m_service.boundingBox = QRectF(QPointF(minLon, minLat), QPointF(maxLon, maxLat));
    }

    // round bounding box coordinates to stabilize the diff gbfs-feeds.json a bit
    constexpr const auto COORD_RESOLUTION = 100.0;
    m_service.boundingBox.setLeft(std::floor(m_service.boundingBox.left() * COORD_RESOLUTION) / COORD_RESOLUTION);
    m_service.boundingBox.setTop(std::floor(m_service.boundingBox.top() * COORD_RESOLUTION) / COORD_RESOLUTION);
    m_service.boundingBox.setRight(std::ceil(m_service.boundingBox.right() * COORD_RESOLUTION) / COORD_RESOLUTION);
    m_service.boundingBox.setBottom(std::ceil(m_service.boundingBox.bottom() * COORD_RESOLUTION) / COORD_RESOLUTION);

    qDebug() << "bounding box:" << m_service.boundingBox;
    GBFSServiceRepository::store(m_service);
    Q_EMIT finished();
}

bool GBFSJob::shouldFetchFile(GBFS::FileType fileType) const
{
    return m_fileTypes.empty() || std::find(m_fileTypes.begin(), m_fileTypes.end(), fileType) != m_fileTypes.end();
}
