From patchwork Sat Nov 2 22:32:28 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lukas Fleischer X-Patchwork-Id: 1327 Return-Path: Delivered-To: patchwork@archlinux.org Received: from apollo.archlinux.org (localhost [127.0.0.1]) by apollo.archlinux.org (Postfix) with ESMTP id 6025914DA3F4D for ; Sat, 2 Nov 2019 22:31:36 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on apollo X-Spam-Level: X-Spam-Status: No, score=-3.4 required=5.0 tests=DKIMWL_WL_HIGH=-0.001, DKIM_SIGNED=0.1,DKIM_VALID=-0.1,DKIM_VALID_AU=-0.1, MAILING_LIST_MULTI=-1,RCVD_IN_DNSWL_MED=-2.3,SPF_HELO_NONE=0.001, T_DMARC_POLICY_NONE=0.01 autolearn=unavailable autolearn_force=no version=3.4.2 X-Spam-BL-Results: [127.0.0.4] [127.0.9.2] Received: from orion.archlinux.org (orion.archlinux.org [IPv6:2a01:4f8:160:6087::1]) by apollo.archlinux.org (Postfix) with ESMTPS for ; Sat, 2 Nov 2019 22:31:36 +0000 (UTC) Received: from orion.archlinux.org (localhost [127.0.0.1]) by orion.archlinux.org (Postfix) with ESMTP id 483AC163F0A713; Sat, 2 Nov 2019 22:31:35 +0000 (UTC) Received: from luna.archlinux.org (luna.archlinux.org [IPv6:2a01:4f8:160:3033::2]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange ECDHE (P-384) server-signature RSA-PSS (4096 bits)) (No client certificate requested) by orion.archlinux.org (Postfix) with ESMTPS; Sat, 2 Nov 2019 22:31:35 +0000 (UTC) Received: from luna.archlinux.org (luna.archlinux.org [127.0.0.1]) by luna.archlinux.org (Postfix) with ESMTP id 37CF62CAB2; Sat, 2 Nov 2019 22:31:35 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=archlinux.org; s=luna2; t=1572733895; bh=FQr3NciTCj4r06CWCAcDpRBRwnGHA6LxgxVi4UC7RbI=; h=From:To:Subject:Date:List-Id:List-Unsubscribe:List-Archive: List-Post:List-Help:List-Subscribe; b=Rn1+1/8fJzvu7N7Nb9QSaA6P+y42rWstsOArZsM9Z1+2qrC6Wv1gnWFhwJYgOWDaO C4qjOtJ8DvwUTeDgi5NJWTQ8zqWTHd2v20BuSYwe0THz1/ZP7DlR0chtM1tmOD5aMQ FIv4BDuYaymSVucuDoX0vXYb7wXKuMFLvC2c9PPHrvL7tWGamybyFO1C5OBXOuVGQ5 cEoU44tQM1fsVD0Tq6AltosJ0WfzwpGVmv/5O8No1fQK3Q0DdDWApjPBrZt31p/lS0 Iyi+hjBVOqBapfNQOUjnI/AANjc9WZeaa4CfGpHuORW9ovUxsHBXfUwDA1GDGS7rq5 c+GaI7TZGUb5ZxZerC3FFU2lJhqH8RZyvhifaIyV0DBdl/f/+fV221wqR2o6g30h4p zilXZ+W79NDhqxToUFTzg307chcIJ32chuXIJf9VAFa0KPe6vCfE/9j78/ABLig+A1 ZhGyQ7JzMRyTi85qDfg8cbTYOqOSHhO3x0qbI82xLdb/cJWstmH5V2bmnrApvy+oip hWKRim59NwDHI/rrUCcL33/lZdFUbB8zcE7tLM1aQEP38101wUJtmvz8q63bkYT4bB Fw/tsmsikWOWE5zT7vnu2v2A/4nQejmLiqALv+yFMYr3KKjMN7ffjCgxQ1GC3/b+IQ ARnbYMA8sEt3exYvK+siQwtY= Received: from luna.archlinux.org (luna.archlinux.org [127.0.0.1]) by luna.archlinux.org (Postfix) with ESMTP id 24A4120954 for ; Sat, 2 Nov 2019 22:31:33 +0000 (UTC) Received: from orion.archlinux.org (orion.archlinux.org [88.198.91.70]) by luna.archlinux.org (Postfix) with ESMTPS for ; Sat, 2 Nov 2019 22:31:33 +0000 (UTC) Received: from orion.archlinux.org (localhost [127.0.0.1]) by orion.archlinux.org (Postfix) with ESMTP id 20521163F0A6EA for ; Sat, 2 Nov 2019 22:31:32 +0000 (UTC) Received: from localhost (unknown [72.138.14.22]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange ECDHE (P-384) server-signature RSA-PSS (4096 bits) server-digest SHA256) (No client certificate requested) (Authenticated sender: lfleischer) by orion.archlinux.org (Postfix) with ESMTPSA id A8CD9163F08DAB for ; Sat, 2 Nov 2019 22:31:31 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=archlinux.org; s=orion; t=1572733892; bh=FQr3NciTCj4r06CWCAcDpRBRwnGHA6LxgxVi4UC7RbI=; h=From:To:Subject:Date; b=i7o6r2V3cEQvwl2eX8llaYb58mrIBcWE2uP2rsf4C5srIxi8DxWfpGXh6X4mIsA5F uQfQpPd6Z0ALudDU3P7L+JeHeQ8CguFKdGmZ103PiJo8/5K95MI1cHid12iQBZwg4T MepSSrh1hgfsu7NIKLNhWkeh1g7O9LVurm3RRn73q0XihRNdZ20MQcCzikfW+R7Upc uIR/ogfhbgZjAj3aJazOdWNCRRFdzYdunubIgBy9/GBhXKLeg1qeUjS6GlxQPeYGbu AX6B4C099Mi6+hXhm39xAuTBoyOobJ0RuIWwXb0346N2LIAxUzFlsIBcBM0GVz47BY RWR0zAx6r5yxXsDofbP9v93fQWrTJuyPgK/CHSt1pe1+s8fAcQi7tY8yBbKiQ0q/Ac KmuYJ1oPGUumw8O2TZeEfhKbZQ/FLWdgmQMvpWsK0vOB1yTTAxzcN9941zWiE2SrUS qxDFV53SuzhMJ1/ls4N4EI30uWcRAP4mLnkRYa9YNzQnk5+nbC/8cWP9YftvIuYDCG C8zO9iJ8u4+kiyM8z2yx255qYVIubVJTbZQ2useWWZn6I2sNQK0p3tBr22PBcuMqMT twbvVGSZ1sAZ40dV3n88SiYbVzzfADqZ5n+E7ro3G38RK3tEsiWyAA2I2ABv7rctbD faiyrl+e23ZYSzdTKJqneYm4= From: Lukas Fleischer To: aur-dev@archlinux.org Subject: [PATCH] aurjson: use APCu/memcached for rate limiting Date: Sat, 2 Nov 2019 18:32:28 -0400 Message-Id: <20191102223228.27031-1-lfleischer@archlinux.org> X-Mailer: git-send-email 2.23.0 MIME-Version: 1.0 X-BeenThere: aur-dev@archlinux.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "Arch User Repository \(AUR\) Development" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: aur-dev-bounces@archlinux.org Sender: "aur-dev" There's no need to use permanent storage for rate limiting information; try to keep it in memory if caching is enabled. From experiments with our live setup, this reduces the number of INSERT/DELETE operations per second from 15 to almost 0. Disk writes on the server hosting the AUR are reduced by 90% (from ~3MB/s to ~300kB/s). Signed-off-by: Lukas Fleischer --- web/lib/aurjson.class.php | 47 ++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/web/lib/aurjson.class.php b/web/lib/aurjson.class.php index 1c31a65..0ac586f 100644 --- a/web/lib/aurjson.class.php +++ b/web/lib/aurjson.class.php @@ -152,23 +152,26 @@ class AurJSON { return false; } - $window_length = config_get("ratelimit", "window_length"); $this->update_ratelimit($ip); - $stmt = $this->dbh->prepare(" - SELECT Requests FROM ApiRateLimit - WHERE IP = :ip"); - $stmt->bindParam(":ip", $ip); - $result = $stmt->execute(); - if (!$result) { - return false; - } + $status = false; + $value = get_cache_value('ratelimit:' . $ip, $status); + if (!$status) { + $stmt = $this->dbh->prepare(" + SELECT Requests FROM ApiRateLimit + WHERE IP = :ip"); + $stmt->bindParam(":ip", $ip); + $result = $stmt->execute(); + + if (!$result) { + return false; + } - $row = $stmt->fetch(PDO::FETCH_ASSOC); - if ($row['Requests'] > $limit) { - return true; + $row = $stmt->fetch(PDO::FETCH_ASSOC); + $value = $row['Requests']; } - return false; + + return $value > $limit; } /* @@ -182,9 +185,23 @@ class AurJSON { $window_length = config_get("ratelimit", "window_length"); $db_backend = config_get("database", "backend"); $time = time(); - - // Clean up old windows $deletion_time = $time - $window_length; + + /* Try to use the cache. */ + $status = false; + $value = get_cache_value('ratelimit-ws:' . $ip, $status); + if (!$status || ($status && $value < $deletion_time)) { + if (set_cache_value('ratelimit-ws:' . $ip, $time, $window_length) && + set_cache_value('ratelimit:' . $ip, 1, $window_length)) { + return; + } + } else { + $value = get_cache_value('ratelimit:' . $ip, $status); + if ($status && set_cache_value('ratelimit:' . $ip, $value + 1, $window_length)) + return; + } + + /* Clean up old windows. */ $stmt = $this->dbh->prepare(" DELETE FROM ApiRateLimit WHERE WindowStart < :time");