Skip to content

Commit 859326b

Browse files
boonebgorgesboonebgorges
committed
First pass at Known Agents AI fetching script.
See cuny-academic-commons/commons-in-a-box#560.
1 parent 6e8000b commit 859326b

3 files changed

Lines changed: 283 additions & 0 deletions

File tree

‎bin/build-known-agents.php‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
<?php
2+
declare(strict_types=1);
3+
4+
require_once __DIR__ . '/classes/KnownAgentsRobotsBuilder.php';
5+
6+
$options = getopt(
7+
'',
8+
[
9+
'token:',
10+
'preset::',
11+
'types::',
12+
'robots-out::',
13+
'php-out:',
14+
'disallow::',
15+
]
16+
);
17+
18+
$token = $options['token'] ?? getenv( 'KNOWN_AGENTS_TOKEN' ) ?: '';
19+
20+
if ( '' === trim( $token ) ) {
21+
fwrite( STDERR, "Missing token. Pass --token=... or set KNOWN_AGENTS_TOKEN.\n" );
22+
exit( 1 );
23+
}
24+
25+
$disallow = $options['disallow'] ?? '/';
26+
27+
if ( ! empty( $options['types'] ) ) {
28+
$agent_types = array_values(
29+
array_filter(
30+
array_map(
31+
'trim',
32+
explode( ',', (string) $options['types'] )
33+
)
34+
)
35+
);
36+
} else {
37+
$preset = $options['preset'] ?? 'training-only';
38+
$agent_types = KnownAgentsRobotsBuilder::get_agent_types_for_preset( (string) $preset );
39+
}
40+
41+
$robots_out = $options['robots-out'] ?? null;
42+
$php_out = $options['php-out'] ?? null;
43+
44+
if ( empty( $php_out ) ) {
45+
fwrite( STDERR, "Missing required --php-out=/path/to/file.php\n" );
46+
exit( 1 );
47+
}
48+
49+
try {
50+
$robots_txt = KnownAgentsRobotsBuilder::fetch_robots_txt( $token, $agent_types, (string) $disallow );
51+
$user_agents = KnownAgentsRobotsBuilder::extract_user_agents( $robots_txt );
52+
$php_file = KnownAgentsRobotsBuilder::build_php_array_file( $user_agents, $agent_types );
53+
54+
if ( is_string( $robots_out ) && '' !== $robots_out ) {
55+
$dir = dirname( $robots_out );
56+
57+
if ( ! is_dir( $dir ) && ! mkdir( $dir, 0777, true ) && ! is_dir( $dir ) ) {
58+
throw new RuntimeException( 'Failed to create robots output directory: ' . $dir );
59+
}
60+
61+
file_put_contents( $robots_out, $robots_txt );
62+
}
63+
64+
$php_dir = dirname( $php_out );
65+
66+
if ( ! is_dir( $php_dir ) && ! mkdir( $php_dir, 0777, true ) && ! is_dir( $php_dir ) ) {
67+
throw new RuntimeException( 'Failed to create PHP output directory: ' . $php_dir );
68+
}
69+
70+
file_put_contents( $php_out, $php_file );
71+
72+
fwrite(
73+
STDOUT,
74+
sprintf(
75+
"Done. Extracted %d user-agent tokens.\n",
76+
count( $user_agents )
77+
)
78+
);
79+
} catch ( Throwable $e ) {
80+
fwrite( STDERR, $e->getMessage() . "\n" );
81+
exit( 1 );
82+
}

‎bin/classes/KnownAgentsRobotsBuilder.php‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
<?php
2+
3+
declare( strict_types=1 );
4+
5+
final class KnownAgentsRobotsBuilder {
6+
private const API_URL = 'https://api.knownagents.com/robots-txts';
7+
8+
/**
9+
* Policy presets mapped to Known Agents agent_types.
10+
*
11+
* @var array<string, array<int, string>>
12+
*/
13+
private const PRESETS = [
14+
'training-only' => [
15+
'AI Data Scraper',
16+
],
17+
'training-plus-undocumented' => [
18+
'AI Data Scraper',
19+
'Undocumented AI Agent',
20+
],
21+
'aggressive' => [
22+
'AI Data Scraper',
23+
'Undocumented AI Agent',
24+
'Scraper',
25+
'Intelligence Gatherer',
26+
],
27+
];
28+
29+
/**
30+
* Fetch robots.txt text from Known Agents.
31+
*
32+
* @param string $token Bearer token for the project.
33+
* @param array<int, string> $agent_types Known Agents categories to request.
34+
* @param string $disallow Path to disallow. Defaults to '/'.
35+
* @return string
36+
*/
37+
public static function fetch_robots_txt( string $token, array $agent_types, string $disallow = '/' ): string {
38+
if ( '' === trim( $token ) ) {
39+
throw new InvalidArgumentException( 'Known Agents token is required.' );
40+
}
41+
42+
if ( empty( $agent_types ) ) {
43+
throw new InvalidArgumentException( 'At least one agent type is required.' );
44+
}
45+
46+
$payload = json_encode(
47+
[
48+
'agent_types' => array_values( $agent_types ),
49+
'disallow' => $disallow,
50+
],
51+
JSON_UNESCAPED_SLASHES
52+
);
53+
54+
if ( false === $payload ) {
55+
throw new RuntimeException( 'Failed to encode Known Agents request payload.' );
56+
}
57+
58+
$ch = curl_init( self::API_URL );
59+
60+
if ( false === $ch ) {
61+
throw new RuntimeException( 'Failed to initialize cURL.' );
62+
}
63+
64+
curl_setopt_array(
65+
$ch,
66+
[
67+
CURLOPT_POST => true,
68+
CURLOPT_RETURNTRANSFER => true,
69+
CURLOPT_HTTPHEADER => [
70+
'Authorization: Bearer ' . $token,
71+
'Content-Type: application/json',
72+
],
73+
CURLOPT_POSTFIELDS => $payload,
74+
CURLOPT_TIMEOUT => 30,
75+
CURLOPT_CONNECTTIMEOUT => 10,
76+
]
77+
);
78+
79+
$response = curl_exec( $ch );
80+
$errno = curl_errno( $ch );
81+
$error = curl_error( $ch );
82+
$code = (int) curl_getinfo( $ch, CURLINFO_RESPONSE_CODE );
83+
84+
curl_close( $ch );
85+
86+
if ( 0 !== $errno ) {
87+
throw new RuntimeException( 'Known Agents request failed: ' . $error );
88+
}
89+
90+
if ( ! is_string( $response ) ) {
91+
throw new RuntimeException( 'Known Agents response was not a string.' );
92+
}
93+
94+
if ( 200 > $code || 300 <= $code ) {
95+
throw new RuntimeException(
96+
sprintf(
97+
'Known Agents API returned HTTP %d. Response: %s',
98+
$code,
99+
$response
100+
)
101+
);
102+
}
103+
104+
return $response;
105+
}
106+
107+
/**
108+
* Extract user-agent tokens from robots.txt text.
109+
*
110+
* @param string $robots_txt
111+
* @return array<int, string>
112+
*/
113+
public static function extract_user_agents( string $robots_txt ): array {
114+
$lines = preg_split( '/\R/', $robots_txt );
115+
116+
if ( false === $lines ) {
117+
return [];
118+
}
119+
120+
$seen = [];
121+
$out = [];
122+
123+
foreach ( $lines as $line ) {
124+
$line = trim( $line );
125+
126+
if ( '' === $line || str_starts_with( $line, '#' ) ) {
127+
continue;
128+
}
129+
130+
if ( ! preg_match( '/^User-agent:\s*(.+)$/i', $line, $matches ) ) {
131+
continue;
132+
}
133+
134+
$token = trim( $matches[1] );
135+
136+
if ( '' === $token || '*' === $token ) {
137+
continue;
138+
}
139+
140+
$key = strtolower( $token );
141+
142+
if ( isset( $seen[ $key ] ) ) {
143+
continue;
144+
}
145+
146+
$seen[ $key ] = true;
147+
$out[] = $token;
148+
}
149+
150+
natcasesort( $out );
151+
152+
return array_values( $out );
153+
}
154+
155+
/**
156+
* Build a PHP file that returns metadata + user agent list.
157+
*
158+
* @param array<int, string> $user_agents
159+
* @param array<int, string> $agent_types
160+
* @return string
161+
*/
162+
public static function build_php_array_file( array $user_agents, array $agent_types ): string {
163+
$export = var_export(
164+
[
165+
'generated_at' => gmdate( 'c' ),
166+
'agent_types' => array_values( $agent_types ),
167+
'user_agents' => array_values( $user_agents ),
168+
],
169+
true
170+
);
171+
172+
return <<<PHP
173+
<?php
174+
declare(strict_types=1);
175+
176+
/**
177+
* Generated file. Do not edit by hand.
178+
*/
179+
180+
return {$export};
181+
182+
PHP;
183+
}
184+
185+
/**
186+
* Convenience helper for presets.
187+
*
188+
* @param string $preset
189+
* @return array<int, string>
190+
*/
191+
public static function get_agent_types_for_preset( string $preset ): array {
192+
if ( ! isset( self::PRESETS[ $preset ] ) ) {
193+
throw new InvalidArgumentException( 'Unknown preset: ' . $preset );
194+
}
195+
196+
return self::PRESETS[ $preset ];
197+
}
198+
}

‎phpcs.xml.dist‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222
<exclude-pattern>lib/bp-customizable-group-categories/includes/imported/*</exclude-pattern>
2323
<exclude-pattern>lib/bp-customizable-group-categories/parts/*</exclude-pattern>
2424

25+
<!-- Exclude scripts -->
26+
<exclude-pattern>bin/*</exclude-pattern>
27+
2528
<!-- Check for PHP cross-version compatibility. -->
2629
<config name="testVersion" value="5.6-"/>
2730

0 commit comments

Comments
 (0)