module

Fetcher

Constants

CODEBERG_RELEASES_PATTERN = /:\/\/(www\.)?codeberg\.org\/[^\/]+\/[^\/]+\/releases/i
DEFAULT_RETRIABLE = ->(ex : Exception) do ex.is_a?(RetriableError) || (transient_error?(ex)) end
EMPTY_FEED_METADATA = FeedMetadata.new
GITHUB_RELEASES_PATTERN = /:\/\/(www\.)?github\.com\/[^\/]+\/[^\/]+\/releases/i
GITLAB_RELEASES_PATTERN = /:\/\/[^\/]+\/[^\/]+\/[^\/]+\/-\/releases/i
MAX_ENTITY_DEFINITIONS = 10

Maximum entity definitions allowed before suspecting entity expansion attack

PATTERN_DRIVERS = {REDDIT_URL_PATTERN => DriverType::Reddit, GITHUB_RELEASES_PATTERN => DriverType::Software, GITLAB_RELEASES_PATTERN => DriverType::Software, CODEBERG_RELEASES_PATTERN => DriverType::Software, YOUTUBE_CHANNEL_PATTERN => DriverType::YouTube}
REDDIT_URL_PATTERN = /:\/\/(www\.)?reddit\.com\/r\//i

Pre-compiled regex patterns for performance

RETRY_JITTER_MAX = 1.1
RETRY_JITTER_MIN = 0.9
YOUTUBE_CHANNEL_PATTERN = /:\/\/(www\.)?youtube\.com\/(channel\/|@|c\/|user\/)/i

Class methods

detect_driver(url : String, headers : HTTP::Headers = ::HTTP::Headers.new, config : RequestConfig = RequestConfig.new) : DriverType
Source
error_result(kind : ErrorKind, message : String, status_code : Int32 | Nil = nil) : Result
Source
error_result(err : Error) : Result
Source
pull(url : String, headers : HTTP::Headers = ::HTTP::Headers.new, limit : Int32 = Config::DEFAULT_LIMIT, config : RequestConfig = RequestConfig.new) : Result
Source
pull(url : String, headers : HTTP::Headers, etag : String | Nil, last_modified : String | Nil, limit : Int32 = Config::DEFAULT_LIMIT, config : RequestConfig = RequestConfig.new) : Result
Source
pull_json_feed(url : String, headers : HTTP::Headers = ::HTTP::Headers.new, limit : Int32 = Config::DEFAULT_LIMIT, config : RequestConfig = RequestConfig.new) : Result
Source
pull_reddit(url : String, headers : HTTP::Headers = ::HTTP::Headers.new, limit : Int32 = Config::DEFAULT_LIMIT, config : RequestConfig = RequestConfig.new) : Result
Source
pull_rss(url : String, headers : HTTP::Headers = ::HTTP::Headers.new, limit : Int32 = Config::DEFAULT_LIMIT, config : RequestConfig = RequestConfig.new) : Result
Source
pull_software(url : String, headers : HTTP::Headers = ::HTTP::Headers.new, limit : Int32 = Config::DEFAULT_LIMIT, config : RequestConfig = RequestConfig.new) : Result
Source
pull_youtube(url : String, headers : HTTP::Headers = ::HTTP::Headers.new, limit : Int32 = Config::DEFAULT_LIMIT, config : RequestConfig = RequestConfig.new) : Result
Source
transient_error?(ex : Exception) : Bool
Source
with_retry(config : RequestConfig, is_retriable : Exception -> Bool = DEFAULT_RETRIABLE, &operation : -> Result) : Result
Source

Macros

define_fetch_error(name, superclass = FetchError)
Source
define_pull_method(driver_method, module_name)
Source

Nested types