Keep Mastodon auto-refresh under the rate limit so posts don't fail with 429

core/include/fastsm/platform/mastodon/mastodon_account.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <atomic>
+#include <cstdint>
 #include <optional>
 #include <string>
 #include <vector>
@@ -99,7 +101,11 @@ class MastodonAccount : public SocialAccount {
 
     const MastodonCredentials& credentials() const { return credentials_; }
 
+    bool background_refresh_allowed() const override;
+
 private:
+    // Every HTTP call goes through here so the rate-limit headers are tracked.
+    net::HttpResponse send(const net::HttpRequest& req);
     // Issues an authenticated request; returns the parsed JSON body on 2xx.
     // `out_status` (optional) receives the HTTP status code regardless.
     bool request(const std::string& method, const std::string& url, const std::string& body,
@@ -128,6 +134,8 @@ class MastodonAccount : public SocialAccount {
     // Set once we learn this instance predates grouped notifications (/api/v2/
     // notifications 404s) so we stop probing v2 and go straight to v1.
     bool grouped_notifs_unsupported_ = false;
+    // Unix time until which background refresh is paused (rate limit nearly spent).
+    std::atomic<std::int64_t> throttled_until_{0};
 };
 
 } // namespace fastsm

core/include/fastsm/platform/social_account.hpp

@@ -226,6 +226,10 @@ class SocialAccount {
     // The controller uses this so each fetch pulls as much as the server allows.
     virtual int max_page_size() const { return 40; }
 
+    // False while the account's API rate limit is nearly spent, so background
+    // auto-refresh backs off and leaves room for the user's own actions.
+    virtual bool background_refresh_allowed() const { return true; }
+
     // Fetch one page. Implementations run synchronously on the worker thread.
     virtual TimelinePage items(const TimelineSource& source, int limit,
                                const PageCursor& cursor) = 0;

core/include/fastsm/session/core_session.hpp

@@ -436,6 +436,7 @@ class CoreSession {
     std::int64_t last_speak_reply_ms_ = 0;
 
     std::atomic<int> auto_refresh_seconds_{0};
+    int refresh_tick_ = 0; // auto-refresh passes so far (core loop only); paces idle tabs
     std::atomic<bool> auto_refresh_running_{true};
     std::thread auto_refresh_thread_;
 

core/src/net/winhttp_client.cpp

@@ -166,11 +166,11 @@ HttpResponse WinHttpClient::send(const HttpRequest& req) {
         req.body.empty() ? WINHTTP_NO_REQUEST_DATA : const_cast<char*>(req.body.data());
     if (!WinHttpSendRequest(request, WINHTTP_NO_ADDITIONAL_HEADERS, 0, body_ptr, body_len, body_len,
                             0)) {
-        res.error = "WinHttpSendRequest failed";
+        res.error = "WinHttpSendRequest failed (" + std::to_string(GetLastError()) + ")";
         return res;
     }
     if (!WinHttpReceiveResponse(request, nullptr)) {
-        res.error = "WinHttpReceiveResponse failed";
+        res.error = "WinHttpReceiveResponse failed (" + std::to_string(GetLastError()) + ")";
         return res;
     }
 

core/src/platform/mastodon/mastodon_account.cpp

@@ -115,6 +115,32 @@ void MastodonAccount::load_configuration() {
     }
 }
 
+net::HttpResponse MastodonAccount::send(const net::HttpRequest& req) {
+    net::HttpResponse res = http_->send(req);
+    // Mastodon allows ~300 calls per 5 minutes per account. Auto-refreshing many
+    // open timelines can spend all of it, and then the user's own posts fail with
+    // 429. When the budget runs low, pause background refresh until the server's
+    // window resets. Only our own instance's headers count (not remote timelines).
+    // ponytail: fixed reserve; scale by X-RateLimit-Limit if servers vary widely.
+    constexpr int kReserve = 100;
+    if (req.url.rfind(credentials_.instance_url, 0) == 0) {
+        const auto remaining = res.header("X-RateLimit-Remaining");
+        if (res.status == 429 || (remaining && std::atoi(remaining->c_str()) < kReserve)) {
+            const auto reset = res.header("X-RateLimit-Reset");
+            const std::int64_t until = (reset ? util::parse_iso8601(*reset) : std::nullopt)
+                                           .value_or(util::now_unix() + 300);
+            if (throttled_until_.exchange(until) < util::now_unix())
+                log::write("rate limit low (remaining=" + remaining.value_or("?") +
+                           "); pausing auto-refresh until " + reset.value_or("+5m"));
+        }
+    }
+    return res;
+}
+
+bool MastodonAccount::background_refresh_allowed() const {
+    return util::now_unix() >= throttled_until_.load();
+}
+
 bool MastodonAccount::request(const std::string& method, const std::string& url,
                               const std::string& body, const std::string& content_type,
                               std::string& out_body, long& out_status) {
@@ -126,9 +152,15 @@ bool MastodonAccount::request(const std::string& method, const std::string& url,
         req.headers.push_back({"Content-Type", content_type});
         req.body = body;
     }
-    const net::HttpResponse res = http_->send(req);
+    const net::HttpResponse res = send(req);
     out_status = res.status;
     out_body = res.body;
+    // Log failures (status + server's error text) so a failed action isn't just
+    // an error earcon with nothing to go on. Query string dropped: it can be long.
+    if (!res.ok() && method != "GET")
+        log::write(method + " " + url.substr(0, url.find('?')) + " failed: status=" +
+                   std::to_string(res.status) + (res.error.empty() ? "" : " " + res.error) +
+                   " body=" + res.body.substr(0, 300));
     return res.ok();
 }
 
@@ -225,7 +257,7 @@ TimelinePage MastodonAccount::items(const TimelineSource& source, int limit,
         req.method = "GET";
         req.url = url;
         req.headers.push_back({"Authorization", "Bearer " + credentials_.access_token});
-        const net::HttpResponse res = http_->send(req);
+        const net::HttpResponse res = send(req);
         if (!res.ok())
             return page;
         try {
@@ -251,7 +283,7 @@ TimelinePage MastodonAccount::items(const TimelineSource& source, int limit,
         req.method = "GET";
         req.url = url;
         req.headers.push_back({"Authorization", "Bearer " + credentials_.access_token});
-        const net::HttpResponse res = http_->send(req);
+        const net::HttpResponse res = send(req);
         if (!res.ok())
             return page;
         try {
@@ -300,7 +332,7 @@ TimelinePage MastodonAccount::items(const TimelineSource& source, int limit,
         net::HttpRequest req;
         req.method = "GET";
         req.url = rurl; // unauthenticated: no Authorization header
-        const net::HttpResponse res = http_->send(req);
+        const net::HttpResponse res = send(req);
         if (!res.ok())
             return page;
         json j;
@@ -336,7 +368,7 @@ TimelinePage MastodonAccount::items(const TimelineSource& source, int limit,
         req.method = "GET";
         req.url = url;
         req.headers.push_back({"Authorization", "Bearer " + credentials_.access_token});
-        const net::HttpResponse res = http_->send(req);
+        const net::HttpResponse res = send(req);
         if (res.status == 404) {
             grouped_notifs_unsupported_ = true; // pre-4.3 instance: use v1 from now on
         } else if (res.ok()) {
@@ -385,7 +417,7 @@ TimelinePage MastodonAccount::items(const TimelineSource& source, int limit,
         req.method = "GET";
         req.url = url;
         req.headers.push_back({"Authorization", "Bearer " + credentials_.access_token});
-        const net::HttpResponse res = http_->send(req);
+        const net::HttpResponse res = send(req);
         if (!res.ok())
             return page;
         try {
@@ -486,7 +518,7 @@ TimelinePage MastodonAccount::items(const TimelineSource& source, int limit,
     req.method = "GET";
     req.url = url;
     req.headers.push_back({"Authorization", "Bearer " + credentials_.access_token});
-    const net::HttpResponse res = http_->send(req);
+    const net::HttpResponse res = send(req);
     if (!res.ok())
         return page;
 
@@ -545,7 +577,7 @@ TimelinePage MastodonAccount::items(const TimelineSource& source, int limit,
         preq.method = "GET";
         preq.url = credentials_.instance_url + path + "?pinned=true&limit=" + std::to_string(limit);
         preq.headers.push_back({"Authorization", "Bearer " + credentials_.access_token});
-        const net::HttpResponse pres = http_->send(preq);
+        const net::HttpResponse pres = send(preq);
         if (pres.ok()) {
             try {
                 const json pj = json::parse(pres.body);
@@ -784,7 +816,7 @@ std::string MastodonAccount::remote_account_id(const std::string& base,
     net::HttpRequest req;
     req.method = "GET";
     req.url = base + "/api/v1/accounts/lookup?acct=" + util::percent_encode(username);
-    net::HttpResponse res = http_->send(req);
+    net::HttpResponse res = send(req);
     if (res.ok()) {
         try {
             if (std::string id = json::parse(res.body).value("id", std::string()); !id.empty())
@@ -794,7 +826,7 @@ std::string MastodonAccount::remote_account_id(const std::string& base,
     }
     // Older servers lack /lookup; fall back to account search.
     req.url = base + "/api/v1/accounts/search?q=" + util::percent_encode(username) + "&limit=1";
-    res = http_->send(req);
+    res = send(req);
     if (res.ok()) {
         try {
             const json arr = json::parse(res.body);
@@ -1084,7 +1116,7 @@ FullRelationResult MastodonAccount::fetch_all_relations(const std::string& id, b
         req.method = "GET";
         req.url = url;
         req.headers.push_back({"Authorization", "Bearer " + credentials_.access_token});
-        const net::HttpResponse res = http_->send(req);
+        const net::HttpResponse res = send(req);
         if (res.status == 429) {
             out.status = FullRelationResult::Status::RateLimited;
             return out;

core/src/session/core_session.cpp

@@ -4271,11 +4271,37 @@ void CoreSession::switch_account(const std::string& new_key) {
 }
 
 void CoreSession::refresh_all_accounts() {
+    // Threads and user timelines with nothing new in a day rarely change, yet with
+    // dozens open, polling each one every tick is what spends the rate limit. Poll
+    // those only every kIdleEvery ticks; the timeline being viewed always refreshes.
+    constexpr int kIdleEvery = 15;
+    const bool idle_turn = ++refresh_tick_ % kIdleEvery == 0;
+    const std::int64_t stale_before = util::now_unix() - 24 * 3600;
+    const TimelineController* viewed =
+        current_ >= 0 && current_ < static_cast<int>(timelines_.size()) ? timelines_[current_].get()
+                                                                        : nullptr;
+    auto idle = [&](const TimelineController& tc) {
+        const auto kind = tc.source().kind;
+        if (&tc == viewed ||
+            (kind != TimelineSource::Kind::Thread && kind != TimelineSource::Kind::UserPosts))
+            return false;
+        std::int64_t newest = 0;
+        for (const auto& it : tc.items())
+            newest = std::max(newest, it.sort_date());
+        return newest > 0 && newest < stale_before;
+    };
+    // Skip accounts near their rate limit so posting and other actions still work.
+    auto refresh = [&](TimelineController& tc) {
+        if (!idle_turn && idle(tc))
+            return;
+        if (!tc.account() || tc.account()->background_refresh_allowed())
+            tc.refresh();
+    };
     for (auto& tc : timelines_)
-        tc->refresh();
+        refresh(*tc);
     for (auto& [key, v] : parked_)
         for (auto& tc : v)
-            tc->refresh();
+            refresh(*tc);
 }
 
 std::unique_ptr<TimelineController> CoreSession::make_controller(SocialAccount* account,

docs/changelog.txt

@@ -4,6 +4,8 @@ FastSMRW changelog
 0.6.0
 -----
 
+- Fixed: on Mastodon, posting no longer fails with an error sound when you have many timelines open; background refreshing now slows down before it uses up your server's request limit.
+- Changed: threads and user timelines with nothing new for a day now check for updates every 15 minutes instead of every minute, unless you're viewing them, so keeping many open no longer strains your server's request limit.
 - New: on iPhone, an Accounts button next to the More button lists your accounts so you can jump straight to one, and add, remove or configure accounts.
 - New: on iPhone, Command+Up and Command+Down jump to the top and bottom of the timeline.
 - Fixed: on iPhone, a two-finger scrub now closes the media player or an open dialog first, instead of always closing the current timeline.

tests/main.cpp

@@ -97,6 +97,7 @@ void test_capi_session_events();
 
 // From test_thread.cpp
 void test_mastodon_thread_fetch();
+void test_mastodon_rate_limit_pauses_refresh();
 void test_mastodon_thread_folding();
 void test_mastodon_instance_max_chars();
 void test_mastodon_user_pinned_posts();
@@ -228,6 +229,7 @@ int main() {
     test_sse_multiline_crlf_comments();
     test_capi_session_events();
     test_mastodon_thread_fetch();
+    test_mastodon_rate_limit_pauses_refresh();
     test_mastodon_thread_folding();
     test_mastodon_instance_max_chars();
     test_mastodon_user_pinned_posts();

tests/test_thread.cpp

@@ -255,3 +255,51 @@ void test_mastodon_grouped_follow_request_fetch() {
     if (fav)
         CHECK(fav->type == Notification::Kind::Favourite);
 }
+
+namespace {
+// Answers every call with an empty page and whatever rate-limit headers are set.
+struct FakeRateLimitHttp : net::IHttpClient {
+    net::Headers headers;
+    long status = 200;
+    net::HttpResponse send(const net::HttpRequest&) override {
+        net::HttpResponse res;
+        res.status = status;
+        res.body = "[]";
+        res.headers = headers;
+        return res;
+    }
+};
+} // namespace
+
+void test_mastodon_rate_limit_pauses_refresh() {
+    FakeRateLimitHttp http;
+    MastodonCredentials cred;
+    cred.instance_url = "https://example.social";
+    cred.access_token = "tok";
+    User me;
+    me.id = "me";
+    MastodonAccount account(cred, me, &http);
+
+    // Plenty of budget left: background refresh keeps running.
+    http.headers = {{"X-RateLimit-Remaining", "250"}, {"X-RateLimit-Reset", "2999-01-01T00:00:00.000Z"}};
+    account.items(TimelineSource::home(), 40, {});
+    CHECK(account.background_refresh_allowed());
+
+    // Nearly spent: paused until the server's reset time.
+    http.headers = {{"X-RateLimit-Remaining", "20"}, {"X-RateLimit-Reset", "2999-01-01T00:00:00.000Z"}};
+    account.items(TimelineSource::home(), 40, {});
+    CHECK(!account.background_refresh_allowed());
+
+    // Once the reset time has passed, refresh resumes.
+    MastodonAccount fresh(cred, me, &http);
+    http.headers = {{"X-RateLimit-Remaining", "0"}, {"X-RateLimit-Reset", "2000-01-01T00:00:00.000Z"}};
+    fresh.items(TimelineSource::home(), 40, {});
+    CHECK(fresh.background_refresh_allowed());
+
+    // A bare 429 (no headers) pauses too.
+    MastodonAccount limited(cred, me, &http);
+    http.headers = {};
+    http.status = 429;
+    limited.items(TimelineSource::home(), 40, {});
+    CHECK(!limited.background_refresh_allowed());
+}