Posted by EvanDonovan on April 20, 2010 at 10:37pm
Last updated by EvanDonovan on Tue, 2010-04-20 22:40
Last updated by EvanDonovan on Tue, 2010-04-20 22:40
Currently this is just a genericized version of my VCL for UrbanMinistry.org, and TechMission's other websites, as I mentioned it in the High Performance BoF yesterday. Later on, I hope to add annotations that explain some of the tips and tricks that are happening here. There are some hints as to what's going on in the comments, though.
#This is a basic VCL configuration file for varnish. See the vcl(7)
#man page for details on VCL syntax and semantics.
#
#Default backend definition. Set this to point to your content
#server.
#
backend default {
.host = "127.0.0.1";
.port = "8008";
.connect_timeout = 600s;
.first_byte_timeout = 600s;
.between_bytes_timeout = 600s;
}
backend lighttpd {
.host = "127.0.0.1";
.port = "81";
.connect_timeout = 600s;
.first_byte_timeout = 600s;
.between_bytes_timeout = 600s;
}
acl techmission_internal {
"localhost";
"127.0.0.1";
}
sub vcl_recv {
// Allow a grace period for offering "stale" data in case backend lags (http://varnish-cache.org/wiki/VCLExampleGrace)
set req.grace = 5m;
// block outside world from our test sites
if ((req.http.host ~ "www.domain1.org|www.domain2.org") && !(client.ip ~ techmission_internal) && !(req.url ~ "^/ad|^/files")) {
error 403 "Forbidden";
}
if((req.url ~ "/server-status" || req.url ~ "/whm-server-status") && !(client.ip ~ techmission_internal)) {
error 404 "Not Found";
}
// add ping url to test Varnish status
if (req.request == "GET" && req.url ~ "/varnish-ping") {
error 200 "OK";
}
/* Normalize host header to reduce variation in cache */
if (req.http.host == "domain.org" && req.url !~ "^/blogs") {
set req.http.host = "www.domain.org";
}
/* Normalize Accept-Encoding to reduce effects of Vary: Accept-Encoding
(cf. http://varnish-cache.org/wiki/FAQ/Compression)
Also note that Vary: User-Agent is considered harmful
(cf. http://www.mail-archive.com/varnish-misc@projects.linpro.no/msg03296.html) */
if (req.http.Accept-Encoding) {
if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
// Don't compress already-compressed files
remove req.http.Accept-Encoding;
}
elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
}
elsif (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
}
else {
// unknown algorithm
remove req.http.Accept-Encoding;
}
}
// Remove has_js and Google Analytics __* cookies. Also remove collapsiblock cookies.
set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(__[a-z]+|__utma_a2a|has_js|collapsiblock)=[^;]*", "");
// Remove JSESSIONID cookie from ChristianVolunteering.org static files and pages that are same for all users
if (req.http.host ~ "christianvolunteering.org" &&
(req.url ~ "^/$" ||
req.url ~ "(searchform|advancedsearch|shorttermmissions|recruitvolunteers|volunteergettingstarted|virtualvolunteer|organizationsearch|abs-bible-outreach|abs-executive-volunteers|abs-traveling-engagement-center|churchinstructions|sitemap|city|virtual|organizationlistings|orglistings7407|technology|volunteerlistings|forgotpassword|churchvolunteer|churchvolunteering|servicetrip|region|citysitemap|searchformadv|personalitytest|groupvolunteering|disasterreliefvolunteering|disasterrelief|internships|christiangapyear|about|FAQs|bookrecommendations|contact|pressrelease|training|volunteerstart|volunteerstories|articles)\.jsp$" ||
req.url ~ "org/org[0-9]+\.jsp$" ||
req.url ~ "org/opp[0-9]+\.jsp$" ||
req.url ~ "orglistings[0-9]+\.jsp$" ||
req.url ~ "org/[^/]+\.jsp$" ||
req.url ~ "volunteer/[^/]+\.jsp$")
) {
set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(JSESSIONID)=[^;]*", "");
}
// Remove a ";" prefix, if present.
set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
// Remove empty cookies.
if (req.http.Cookie ~ "^\s*$") {
unset req.http.Cookie;
}
// exclude umjobs and gospelpedia test sites
if (req.http.host ~ "domain1.org" || req.http.host ~ "domain2.org") {
return (pass);
}
// exclude the cron and supercron pages
if (req.url ~ "cron.php") {
return (pass);
}
// exclude dynamic pages (as did Boost)
if (req.http.host ~ "domain.org" && req.url ~ "^/(user/login|user/password|user/register|logout|cart|post-blog|site-feedback|cgi-bin/webscr|redirect-home|cv-enroll|recommended-content|node/8755|node/8830|node/8351|node/8757|node/8831|cart/(.*)|uc_paypal/(.*)|civicrm/(.*)|admin/(.*)|recommended-content/(.*)|comment/reply/(.*)|node/add/(.*))" ) {
return (pass);
}
// exclude in case of Referer Theme
if (req.http.host ~ "domain.org" && req.http.referer ~ "www.christianvolunteering.org|worldvision.christianvolunteering.org|ccda.christianvolunteering.org|www.ccda.org|www.urbanresource.net|mobile.urbanministry.org|www.ministeriourbano.com") {
return (pass);
}
/* Rules to fix Moodle (thanks, gchaix!) */
// Cache Moodle theme files
if (req.url ~ "/pix/.*\.gif$") {
return (lookup);
}
// Moodle doesn't like to be cached, passing
if (req.http.Cookie ~ "(MoodleSession|MoodleSessionTest)") {
return (pass);
}
if (req.http.host == "www.domain.edu" && req.url ~ "^/courses") {
return (pass);
}
if (req.url ~ "file.php") {
return (pass);
}
// WPMU themes are not playing well with static file caching
if (req.http.host == "domain.org" && req.url ~ "/blogs/(.*)/wp-content/themes") {
return (pass);
}
/* Rules for static file caching */
// static files get served by Lighttpd
if (req.http.host != "server2.techmission.org" && req.url ~ "\.(gif|jpg|swf|css|js|png|jpg|jpeg|gif|png|tiff|tif|svg|swf|ico|doc|ppt|pps|xls|odc|odb|odf|odg|odi|odp|ods|odt|sxc|sxd|sxi|sxw)$") {
// Lighttpd does not require cookies
unset req.http.Cookie;
unset req.http.Authorization;
set req.backend = lighttpd;
}
// large media files get piped directly to lighttpd, to avoid overfilling cache
if (req.url ~ "\.(mp3|mp4|m4a|ogg|mov|avi|wmv)$" && req.url !~ "audio/download") {
set req.backend = lighttpd;
pipe;
}
// pipe large media files that come from Drupal, also, but they can't go to lighty
if (req.url ~ "audio/play" || req.url ~ "audio/download") {
pipe;
}
}
sub vcl_hash {
if (req.http.Cookie) {
set req.hash += req.http.Cookie;
}
/* Have a separate object cache for mobile site based on User-Agent */
if (req.http.host == "www.domain.org" && req.http.User-Agent ~ "(iPhone|iPod)") {
set req.hash += "mobile";
}
}
sub vcl_fetch {
// Grace to allow varnish to serve content if backend is lagged
set obj.grace = 5m;
// Add line showing what cookie is once stripped by regex in vcl_recv
set obj.http.X-Stripped-Cookie = req.http.Cookie;
set obj.http.X-Request-URL = req.url;
/* removing Set-Cookie headers that prevent caching */
// Don't have cookies on static files (gchaix says may cause loss of session; I haven't observed that)
if (req.url ~ "\.(gif|jpg|swf|css|js|png|jpg|jpeg|gif|png|tiff|tif|svg|swf|ico|doc|ppt|pps|xls|odc|odb|odf|odg|odi|odp|ods|odt|sxc|sxd|sxi|sxw)$") {
remove obj.http.Set-Cookie;
}
// Don't set session cookie on ChristianVolunteering.org static files or pages that are same for all users
if (req.http.host ~ "christianvolunteering.org" &&
(req.url ~ "^/$" ||
req.url ~ "(searchform|advancedsearch|shorttermmissions|recruitvolunteers|volunteergettingstarted|virtualvolunteer|organizationsearch|abs-bible-outreach|abs-executive-volunteers|abs-traveling-engagement-center|churchinstructions|sitemap|city|virtual|organizationlistings|orglistings7407|technology|volunteerlistings|forgotpassword|churchvolunteer|churchvolunteering|servicetrip|region|citysitemap|searchformadv|personalitytest|groupvolunteering|disasterreliefvolunteering|disasterrelief|internships|christiangapyear|about|FAQs|bookrecommendations|contact|pressrelease|training|volunteerstart|volunteerstories|articles)\.jsp$" ||
req.url ~ "org/org[0-9]+\.jsp$" ||
req.url ~ "org/opp[0-9]+\.jsp$" ||
req.url ~ "orglistings[0-9]+\.jsp$" ||
req.url ~ "org/[^/]+\.jsp$" ||
req.url ~ "volunteer/[^/]+\.jsp$")
) {
set obj.http.Set-Cookie = regsuball(req.http.Cookie, "(^|;\s*)(JSESSIONID)=[^;]*", "");
// Remove empty set-cookie.
if (obj.http.Set-Cookie ~ "^\s*$") {
unset obj.http.Set-Cookie;
}
}
/* ttl extensions */
// If on www.urbanministry.org or static.urbanministry.org, extend TTL by default (pt. 1)
if (req.http.host == "www.domain.org" || req.http.host == "static.domain.org") {
set obj.http.X-TTL-Extend = "YES";
}
// If on cityvision.edu, but not in Moodle, or if on blazinggrace.org, but not in forums, or if on techmission.org, change obj.ttl
if ((req.http.host ~ "domain.edu" && req.url !~ "/courses") || (req.http.host ~ "blazinggrace.org" && req.url !~ "/forums") || (req.http.host ~ "techmission.org")) {
set obj.ttl = 7d;
set obj.http.X-Extended-TTL = "7d";
}
if (obj.status == 404) {
set obj.ttl = 1s;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
/* debugging of why a page was not cacheable */
if (!obj.cacheable) {
set obj.http.X-Cacheable = "NO: Varnish says not cacheable " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
// exclude umjobs and gospelpedia test sites
if (req.http.host ~ "domain1.org" || req.http.host ~ "domain2.org") {
set obj.http.X-Cacheable = "NO: Test domain " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
if (obj.http.Set-Cookie) {
set obj.http.X-Cacheable = "NO: Set-Cookie " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
if (req.http.Cookie ~ "DRUPAL_UID|SESS") {
set obj.http.X-Cacheable = "NO: Got Session " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
if (obj.http.Cache-Control ~ "private" || obj.http.Cache-Control ~ "no-cache") {
set obj.http.X-Cacheable = "NO: Cache-Control set to not cache " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
if (req.url ~ "cron.php") {
set obj.http.X-Cacheable = "NO: Cron job " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
if (req.http.host ~ "domain.org" && req.url ~ "^/(user/login|user/password|user/register|logout|cart|post-blog|site-feedback|cgi-bin/webscr|redirect-home|cv-enroll|recommended-content|node/8755|node/8830|node/8351|node/8757|node/8831|cart/(.*)|uc_paypal/(.*)|civicrm/(.*)|admin/(.*)|recommended-content/(.*)|comment/reply/(.*)|node/add/(.*))" ) {
set obj.http.X-Cacheable = "NO: Drupal un-cacheable path " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
if (req.http.host ~ "domain.org" && req.http.referer ~ "www.christianvolunteering.org|worldvision.christianvolunteering.org|ccda.christianvolunteering.org|www.ccda.org|www.urbanresource.net|mobile.urbanministry.org|www.ministeriourbano.com") {
set obj.http.X-Cacheable = "NO: Referer Theme " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
if (req.request == "POST") {
set obj.http.X-Cacheable = "NO: POST request " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
if (req.http.Authorization) {
set obj.http.X-Cacheable = "NO: HTTP Authentication " obj.http.X-Cacheable;
if (obj.http.X-TTL-Extend) { unset obj.http.X-TTL-Extend; }
}
// extend TTL for urbanministry.org objects (but not panels, views, or quicktabs); invalidation thru varnish.module + um_common.module
if((req.http.host == "www.domain.org" || req.http.host == "static.domain.org") && obj.http.X-TTL-Extend == "YES" && !obj.http.X-Cache-Type) {
set obj.ttl = 7d;
set obj.http.X-Extended-TTL = "7d";
}
}
sub vcl_deliver {
# return (deliver);
// add cache hit data
if (obj.hits > 0) {
// if hit add hit count
set resp.http.X-Cache = "HIT";
set resp.http.X-Cache-Hits = obj.hits;
// set resp.http.X-Cache-Served-URL = "SERVED " obj.http.X-Request-URL; // http headers are apparently not accessible in vcl_deliver
// set resp.http.X-Cache-TTL = obj.ttl; // string representation not implemented yet (currently on 2.0.5)
}
else {
set resp.http.X-Cache = "MISS";
}
}
/* custom error subroutine - to be a little friendlier to our users */
sub vcl_error {
if(obj.status == 503) {
set obj.http.Content-Type = "text/html; charset=utf-8";
synthetic {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
<head>
<title>"} obj.status " " obj.response {"</title>
</head>
<body>
<h1>Error "} obj.status " " obj.response {"</h1>
<p>"} obj.response {"</p>
<p>Sorry we missed you!</p>
<p>We are currently upgrading our websites to serve you better. We should be up again soon.</p>
<p>If you still receive this message 30 minutes from now, please email webmaster@techmission.org.</p>
<h3>Guru Meditation:</h3>
<p>XID: "} req.xid {"</p>
<hr>
<address>
Served by <a href="http://www.varnish-cache.org/">Varnish cache server</a>
</address>
</body>
</html>
"};
return (deliver);
}
elsif(obj.status == 403) {
set obj.http.Content-Type = "text/html; charset=utf-8";
synthetic {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
<head>
<title>"} obj.status " " obj.response {"</title>
</head>
<body>
<h1>Error "} obj.status " " obj.response {"</h1>
<p>"} obj.response {"</p>
<h2>TechMission Developer Access Only</h2>
<p>This page is only accessible to our staff. Please visit our main websites www.techmission.org, www.urbanministry.org, and www.christianvolunteering.org.</p>
<!-- (If you should have access to this page, edit the VCL file to grant yourself access (by adding your current IP to the ACL) and then reload the VCL.) -->
<h3>Guru Meditation:</h3>
<p>XID: "} req.xid {"</p>
<hr>
<address>
Served by <a href="http://www.varnish-cache.org/">Varnish cache server</a>
</address>
</body>
</html>
"};
return (deliver);
}
else {
set obj.http.Content-Type = "text/html; charset=utf-8";
synthetic {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
<head>
<title>"} obj.status " " obj.response {"</title>
</head>
<body>
<h1>Error "} obj.status " " obj.response {"</h1>
<p>"} obj.response {"</p>
<h3>Guru Meditation:</h3>
<p>XID: "} req.xid {"</p>
<hr>
<address>
Served by <a href="http://www.varnish-cache.org/">Varnish cache server</a>
</address>
</body>
</html>
"};
return (deliver);
}
}
#
#Below is a commented-out copy of the default VCL logic. If you
#redefine any of these subroutines, the built-in logic will be
#appended to your code.
#
#sub vcl_recv {
# if (req.request != "GET" &&
# req.request != "HEAD" &&
# req.request != "PUT" &&
# req.request != "POST" &&
# req.request != "TRACE" &&
# req.request != "OPTIONS" &&
# req.request != "DELETE") {
# /* Non-RFC2616 or CONNECT which is weird. */
# return (pipe);
# }
# if (req.request != "GET" && req.request != "HEAD") {
# /* We only deal with GET and HEAD by default */
# return (pass);
# }
# if (req.http.Authorization || req.http.Cookie) {
# /* Not cacheable by default */
# return (pass);
# }
# return (lookup);
#}
#
#sub vcl_pipe {
# # Note that only the first request to the backend will have
# # X-Forwarded-For set. If you use X-Forwarded-For and want to
# # have it set for all requests, make sure to have:
# # set req.http.connection = "close";
# # here. It is not set by default as it might break some broken web
# # applications, like IIS with NTLM authentication.
# return (pipe);
#}
#
#sub vcl_pass {
# return (pass);
#}
#
#sub vcl_hash {
# set req.hash += req.url;
# if (req.http.host) {
# set req.hash += req.http.host;
# } else {
# set req.hash += server.ip;
# }
# return (hash);
#}
#
#sub vcl_hit {
# if (!obj.cacheable) {
# return (pass);
# }
# return (deliver);
#}
#
#sub vcl_miss {
# return (fetch);
#}
#
#sub vcl_fetch {
# if (!obj.cacheable) {
# return (pass);
# }
# if (obj.http.Set-Cookie) {
# return (pass);
# }
# set obj.prefetch = -30s;
# return (deliver);
#}
#
#sub vcl_deliver {
# return (deliver);
#}
#
#sub vcl_discard {
# /* XXX: Do not redefine vcl_discard{}, it is not yet supported */
# return (discard);
#}
#
#sub vcl_prefetch {
# /* XXX: Do not redefine vcl_prefetch{}, it is not yet supported */
# return (fetch);
#}
#
#sub vcl_timeout {
# /* XXX: Do not redefine vcl_timeout{}, it is not yet supported */
# return (discard);
#}
#
#sub vcl_error {
# set obj.http.Content-Type = "text/html; charset=utf-8";
# synthetic {"
#<?xml version="1.0" encoding="utf-8"?>
#<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
# "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
#<html>
# <head>
# <title>"} obj.status " " obj.response {"</title>
# </head>
# <body>
# <h1>Error "} obj.status " " obj.response {"</h1>
# <p>"} obj.response {"</p>
# <h3>Guru Meditation:</h3>
# <p>XID: "} req.xid {"</p>
# <hr>
# <address>
# <a href="http://www.varnish-cache.org/">Varnish cache server</a>
# </address>
# </body>
#</html>
#"};
# return (deliver);
#}