Merge branch 'release/0.2.0'

.travis.yml

@@ -1,6 +1,5 @@
 language: python
 python:
-  - 2.5
   - 2.6
   - 2.7
 

CHANGES.md

@@ -0,0 +1,22 @@
+### 0.2.0 (2014-10-21 12:36:50 UTC)
+
+[full changelog](https://github.com/SickragePVR/SickRage/compare/release_0.1.0...release_0.2.0)
+
+* Fix for failed episodes not counted in total
+* Fix for custom newznab providers with leading integer in name
+* Add checkbox to control proxying of indexers
+* Fix crash on general settings page when git output is None
+* Add subcentre subtitle provider
+* Add return code from hardlinking error to log
+* Fix ABD regex for certain filenames
+* Miscellaneous UI fixes
+* Update Tornado webserver to 4.1dev1 and add the certifi lib dependency.
+* Fix trending shows page from loading full size poster images
+* Add "Archive on first match" to Manage, Mass Update, Edit Selected page
+* Fix searching IPTorrentsProvider
+* Remove travisci python 2.5 build testing
+
+
+### 0.1.0 (2014-10-16 12:35:15 UTC)
+
+* Initial release

TODO.txt

@@ -1,10 +1,10 @@
 2014-10-02
--Fix fuzzy dates on new skin
+-[done] Fix fuzzy dates on new skin
 -Move the sql from the *.tmpl files
 -Add anidb indexer
 -Add supplemental data from anidb during postprocessing
 -Fix grabbing non-propers as propers for certain air-by-date shows
--Allow customization of skin colours
+-[done] Allow customization of skin colours
 -Add prefer torrents/usenet and allow custom delay between grabs from torrents/usenet
 -Better postprocessing handling for torrents
 -[DONE] Add global required words
@@ -16,3 +16,7 @@
 
 2014-10-08
 -Add login page for http auth as opposed to browser dialog box
+
+2014-10-13
+-Fix broken backlog
+-Fix season searches

gui/slick/css/default.css

@@ -386,7 +386,7 @@ a.inner {
 #header .wrapper {
 	margin: 0 auto;
 	position: relative;
-	width: 1250px;
+	width: 1024px;
 }
 
 #header a:hover {
@@ -396,7 +396,7 @@ a.inner {
 #header .showInfo .checkboxControls {
 	margin: 0 auto;
 	position: relative;
-	width: 1250px;
+	width: 1024px;
 }
 
 #logo {
@@ -493,7 +493,7 @@ h2 {
 }
 
 .h2footer {
-	margin: -35px 5px 6px 0px;
+	margin: -35px 0px 6px 0px;
 }
 
 .h2footer select {
@@ -606,7 +606,6 @@ input:not(.btn){
 
 .footer {
 	clear: both;
-	width: 1250px;
 	text-align: center;
 	padding-top: 5px;
 	padding-bottom: 5px;
@@ -1148,7 +1147,7 @@ div#summary tr td {
 	line-height: 24px;
 	margin: 0 auto;
 	padding: 105px 0 0;
-	width: 1250px;
+	width: 1024px;
 }
 
 #content960 {
@@ -2431,7 +2430,7 @@ span.imdbstars > * {
 }
 
 #showCol {
-	width: 950px;
+	width: 730px;
 	float: right;
 	height: 390px;
 	position: relative;
@@ -2468,7 +2467,7 @@ ul.tags li a{
    	-moz-border-radius: 10px;
 	-webkit-border-radius: 10px;
 	border-radius: 10px;
-	padding: 5px 10px;
+	padding: 5px 0 0 10px;
 	margin-bottom: -3px;
 	text-shadow: 0px 1px rgba(0, 0, 0, 0.8);
 	background: #333;
@@ -4976,7 +4975,7 @@ posterlist.css
 }
 
 .show {
-    margin: 10px;
+    margin: 8px;
     background-color: #333;
     border-radius: 3px;
 	width: 186px;
@@ -5067,7 +5066,7 @@ posterlist.css
 	font-size: 11px;
 	text-align: left;
 	display: block;
-	line-height: 18px;
+	line-height: 13px;
 }
 
 .show-status {
@@ -5075,7 +5074,7 @@ posterlist.css
 	font-size: 11px;
 	text-align: left;
 	display: block;
-	line-height: 34px;
+	line-height: 15px;
 }
 
 .show-network-image {
@@ -5086,9 +5085,9 @@ posterlist.css
 .show-dlstats {
 	text-shadow: 1px 1px #000;
 	font-size: 11px;
-	text-align: right;
+	text-align: left;
 	display: block;
-	line-height: 18px;
+	line-height: 15px;
 }
 
 #sort-by {

gui/slick/interfaces/default/comingEpisodes.tmpl

@@ -24,11 +24,13 @@
 </style>
 
 <div class="h2footer align-right">
+#if $layout != 'calendar':
     <b>Key:</b> 
     <span class="listing_overdue">Missed</span>
     <span class="listing_current">Current</span>
     <span class="listing_default">Future</span>
     <span class="listing_toofar">Distant</span>
+#end if
 	<a class="btn forceBacklog" href="webcal://$sbHost:$sbHttpPort/calendar">
 	<i class="icon-calendar icon-white"></i>Subscribe</a>    
     <!-- <span class="listing_unknown">Unknown</span> //-->
@@ -245,6 +247,20 @@
 			});
 		});
 	});
+
+    #set $fuzzydate = 'airdate'
+    #if $sickbeard.FUZZY_DATING:
+    fuzzyMoment({
+        dtInline : true,
+        dtGlue : ' at ',
+        containerClass : '.${fuzzydate}',
+        dateHasTime : true,
+        dateFormat : '${sickbeard.DATE_PRESET}',
+        timeFormat : '${sickbeard.TIME_PRESET}',
+        trimZero : #if $sickbeard.TRIM_ZERO then "true" else "false"#
+    });
+    #end if
+
 });
 //-->
 </script>
@@ -388,10 +404,11 @@
 					</span>
 				</div>
 			
-				<span class="title">Next Episode:</span> <span><%="S%02i" % int(cur_result["season"])+"E%02i" % int(cur_result["episode"]) %> - $cur_result["name"] ($sbdatetime.sbdatetime.fromtimestamp($time.mktime($cur_result["localtime"].timetuple())).sbfdate().decode($sickbeard.SYS_ENCODING))</span>
+				<span class="title">Next Episode:</span> <span><%="S%02i" % int(cur_result["season"])+"E%02i" % int(cur_result["episode"]) %> - $cur_result["name"]</span>
             
 				<div class="clearfix">
-					<span class="title">Airs:</span> <span>$sbdatetime.sbdatetime.fromtimestamp($time.mktime($cur_result["localtime"].timetuple())).sbftime().decode($sickbeard.SYS_ENCODING) on $cur_result["network"]</span>
+
+					<span class="title">Airs: </span><span class="${fuzzydate}">$sbdatetime.sbdatetime.sbfdatetime($cur_result["localtime"]).decode($sickbeard.SYS_ENCODING)</span><span> on $cur_result["network"]</span>
 				</div>
 			
 				<div class="clearfix">
@@ -434,29 +451,43 @@
 
 <input type="hidden" id="sbRoot" value="$sbRoot" />
     #for $day in $dates
-        <table class="sickbeardTable tablesorter" cellspacing="0" border="0" cellpadding="0" style="float:left;width:178px;white-space: nowrap; table-layout: fixed;">
+        <table class="sickbeardTable tablesorter" cellspacing="0" border="0" cellpadding="0" style="float:left;width:146px;white-space: nowrap; table-layout: fixed; background-color:rgb(51,51,51)">
         <thead><tr><th>$day.strftime("%A").decode($sickbeard.SYS_ENCODING).capitalize()</th></tr></thead>
         <tbody>
-
+        #set $day_has_show = False
         #for $cur_result in $sql_results:
             #set $cur_indexer = int($cur_result["indexer"])
             #set $runtime = $cur_result["runtime"]
             #set $airday = $cur_result["localtime"].date()
 
             #if $airday == $day:
+                #set $day_has_show = True
+                #set $airtime = $sbdatetime.sbdatetime.fromtimestamp($time.mktime($cur_result["localtime"].timetuple())).sbftime().decode($sickbeard.SYS_ENCODING)
+                #if $sickbeard.TRIM_ZERO:
+                #set $airtime = re.sub(r"0(\d:\d\d)", r"\1", $airtime, 0, re.IGNORECASE | re.MULTILINE)
+                #end if
 
                 <tr>
-                <td style="overflow: hidden; text-overflow: ellipsis;  font-size: 12px;  LINE-HEIGHT:14px;">
+                    <td style="padding:0">
-                        <a href="$sbRoot/home/displayShow?show=${cur_result["showid"]}"><img alt="" src="$sbRoot/showPoster/?show=${cur_result["showid"]}&amp;which=poster_thumb" width="155" style="padding-bottom: 5px;" /></a>
+                        <div>
-                        <br> $cur_result["localtime"].strftime("%H:%M") on $cur_result["network"]
+                            <a title="${cur_result["show_name"]}" href="$sbRoot/home/displayShow?show=${cur_result["showid"]}"><img alt="" src="$sbRoot/showPoster/?show=${cur_result["showid"]}&amp;which=poster_thumb" width="144" style="padding-bottom: 5px;" /></a>
-                        #set $episodestring = "%sx%s %s" % ($cur_result["season"], $cur_result["episode"], $cur_result["name"])
+                        </div>
-                        <br> <%="S%02i" % int(cur_result["season"])+"E%02i" % int(cur_result["episode"]) %> - $cur_result["name"]
+                        <div class="show-status" style="padding:0 5px 10px 5px">
-                    </td>
+                            <span style="overflow: hidden; text-overflow: ellipsis; display:block">
+                                ${airtime} on $cur_result["network"]
+                            </span>
+                            <span style="overflow: hidden; text-overflow: ellipsis; display:block"  title="$cur_result["name"]">
+                                <%= "S%02i" % int(cur_result["season"]) + "E%02i" % int(cur_result["episode"]) %> - $cur_result["name"]
+                            </span>
+                        </div>
+                    </td> <!-- end $cur_result["show_name"] //-->
                 </tr>
             #end if
 
-        <!-- end $cur_result["show_name"] //-->
         #end for
+        #if not $day_has_show:
+            <tr><td style="padding:0"><span class="show-status" style="padding:5px 10px 10px; text-align:center">No shows for this day</span></td></tr>
+        #end if
         </tbody>
         </table>
     #end for

gui/slick/interfaces/default/config_general.tmpl

@@ -419,6 +419,13 @@
                                 <span class="component-desc">Proxy to use for connecting to providers.  Leave empty to not use proxy.</span>
                             </label>
                         </div>
+						<div class="field-pair">
+							<input type="checkbox" name="proxy_indexers" id="proxy_indexers" #if $sickbeard.PROXY_INDEXERS == True then "checked=\"checked\"" else ""#/>
+							<label class="clearfix" for="proxy_indexers">
+								<span class="component-title">Proxy Indexers</span>
+								<span class="component-desc">Use the proxy for connecting to indexers (thetvdb, tvrage)</span>
+							</label>
+						</div>
 
                         <input type="submit" class="btn config_submitter" value="Save Changes" />
                     </fieldset>

gui/slick/interfaces/default/displayShow.tmpl

@@ -194,7 +194,7 @@
 	</div>
 	
 	<div class="showsummary">
-		<table style="width:83%; float: left;">
+		<table style="width:73%; float: left;">
 		#if $show.network and $show.airs:
 			<tr><td class="showLegend">Originally Airs: </td><td>$show.airs #if not $network_timezones.test_timeformat($show.airs) then " <font color='#FF0000'><b>(invalid Timeformat)</b></font> " else ""# on $show.network</td></tr>
 		#else if $show.network:
@@ -243,7 +243,7 @@
 
 		</table>
 	
-		<table style="width:17%; float: right; vertical-align: middle; height: 100%;">
+		<table style="width:27%; float: right; vertical-align: middle; height: 100%;">
 			<tr><td class="showLegend">Info Language:</td><td><img src="$sbRoot/images/flags/${show.lang}.png" width="16" height="11" alt="$show.lang" title="$show.lang" /></td></tr>
 			#if $sickbeard.USE_SUBTITLES
 			<tr><td class="showLegend">Subtitles: </td><td><img src="$sbRoot/images/#if int($show.subtitles) == 1 then "yes16.png\" alt=\"Y" else "no16.png\" alt=\"N"#" width="16" height="16" /></td></tr>
@@ -279,17 +279,17 @@
 		<input type="hidden" id="indexer" value="$show.indexer" />
 		<input class="btn" type="button" id="changeStatus" value="Go" />
 	</div>
-
-	<div class="float-right clearfix" id="checkboxControls" style="display:inline-block; vertical-align:baseline;">
-		<label for="wanted"><span class="wanted"><input type="checkbox" id="wanted" checked="checked" /> Wanted: <b>$epCounts[$Overview.WANTED]</b></span></label>
-		<label for="qual"><span class="qual"><input type="checkbox" id="qual" checked="checked" /> Low Quality: <b>$epCounts[$Overview.QUAL]</b></span></label>
-		<label for="good"><span class="good"><input type="checkbox" id="good" checked="checked" /> Downloaded: <b>$epCounts[$Overview.GOOD]</b></span></label>
-		<label for="skipped"><span class="skipped"><input type="checkbox" id="skipped" checked="checked" /> Skipped: <b>$epCounts[$Overview.SKIPPED]</b></span></label>
-		<label for="snatched"><span class="snatched"><input type="checkbox" id="snatched" checked="checked" /> Snatched: <b>$epCounts[$Overview.SNATCHED]</b></span></label>
-    </div>
 </div>
 
-<div class="float-right" style="margin-top: -4px;">
+<div id="checkboxControls" style="display:inline-block; vertical-align:baseline;">
+   Filters: <label for="wanted"><span class="wanted"><input type="checkbox" id="wanted" checked="checked" /> Wanted: <b>$epCounts[$Overview.WANTED]</b></span></label>
+    <label for="qual"><span class="qual"><input type="checkbox" id="qual" checked="checked" /> Low Quality: <b>$epCounts[$Overview.QUAL]</b></span></label>
+    <label for="good"><span class="good"><input type="checkbox" id="good" checked="checked" /> Downloaded: <b>$epCounts[$Overview.GOOD]</b></span></label>
+    <label for="skipped"><span class="skipped"><input type="checkbox" id="skipped" checked="checked" /> Skipped: <b>$epCounts[$Overview.SKIPPED]</b></span></label>
+    <label for="snatched"><span class="snatched"><input type="checkbox" id="snatched" checked="checked" /> Snatched: <b>$epCounts[$Overview.SNATCHED]</b></span></label>
+</div>
+
+<div style="margin-top: -4px; display:inline-block; vertical-align:middle">
     <button class="btn btn-mini seriesCheck" style="line-height: 10px;">Select Filtered Episodes</button>
     <button class="btn btn-mini clearAll" style="line-height: 10px;">Clear All</button>
 </div>

gui/slick/interfaces/default/history.tmpl

@@ -186,7 +186,7 @@
 		#for $hItem in $compactResults:
 			<tr>
 				#set $curdatetime = $datetime.datetime.strptime(str($hItem["actions"][0]["time"]), $history.dateFormat)
-				<td class="nowrap">$sbdatetime.sbdatetime.sbfdatetime($curdatetime, show_seconds=True)<span class="sort_data">$time.mktime($curdatetime.timetuple())</span></td>
+				<td class="nowrap"><div class="${fuzzydate}">$sbdatetime.sbdatetime.sbfdatetime($curdatetime, show_seconds=True)</div><span class="sort_data">$time.mktime($curdatetime.timetuple())</span></td>
 				<td width="25%">
 					<span><a style="color: #fff; text-align: center;" href="$sbRoot/home/displayShow?show=$hItem["show_id"]#season-$hItem["season"]">$hItem["show_name"] - <%="S%02i" % int(hItem["season"])+"E%02i" % int(hItem["episode"]) %>#if "proper" in $hItem["resource"].lower or "repack" in $hItem["resource"].lower then ' <span class="quality Proper">Proper</span>' else ""#</a></span>
 				</td>

gui/slick/interfaces/default/home.tmpl

@@ -25,7 +25,7 @@
 #set $sql_statement += '(SELECT COUNT(*) FROM tv_episodes WHERE showid=tv_eps.showid AND season > 0 AND episode > 0 AND airdate > 1 AND status IN ' + $status_download + ') AS ep_downloaded, '
 
 #set $sql_statement += '(SELECT COUNT(*) FROM tv_episodes WHERE showid=tv_eps.showid AND season > 0 AND episode > 0 AND airdate > 1 '
-#set $sql_statement += ' AND ((airdate <= ' + $today + ' AND (status = ' + str($SKIPPED) + ' OR status = ' + str($WANTED) + ')) '
+#set $sql_statement += ' AND ((airdate <= ' + $today + ' AND (status = ' + str($SKIPPED) + ' OR status = ' + str($WANTED) + ' OR status = ' + str($FAILED) + ')) '
 #set $sql_statement += ' OR (status IN ' + status_quality + ') OR (status IN ' + status_download + '))) AS ep_total, '
 
 #set $sql_statement += ' (SELECT airdate FROM tv_episodes WHERE showid=tv_eps.showid AND airdate >= ' + $today + ' AND (status = ' + str($UNAIRED) + ' OR status = ' + str($WANTED) + ') ORDER BY airdate ASC LIMIT 1) AS ep_airs_next '
@@ -247,6 +247,7 @@ function invertSort(){
     #set $fuzzydate = 'airdate'
     #if $sickbeard.FUZZY_DATING:
     fuzzyMoment({
+        dtInline : #if $layout == 'poster' then "true" else "false"#,
         containerClass : '.${fuzzydate}',
         dateHasTime : false,
         dateFormat : '${sickbeard.DATE_PRESET}',
@@ -365,14 +366,14 @@ $myShowList.sort(lambda x, y: cmp(x.name, y.name))
 #set $progressbar_percent = $nom * 100 / $den
 	
 	#if "a" in sickbeard.DATE_PRESET:
-		#set $showheight = "344px"
+		#set $showheight = "334px"
-		#set $tableheight = "62px"
+		#set $tableheight = "52px"
 	#else if "B" in sickbeard.DATE_PRESET:
 		#set $showheight = "374px"
 		#set $tableheight = "92px"
 	#else if "A" in sickbeard.DATE_PRESET:
-		#set $showheight = "344px"
+		#set $showheight = "334px"
-		#set $tableheight = "62px"
+		#set $tableheight = "52px"
 	#else
 		#set $showheight = "323px"
 		#set $tableheight = "42px"
@@ -410,15 +411,14 @@ $myShowList.sort(lambda x, y: cmp(x.name, y.name))
 			</script>
 		
 		<table width="184px" height="$tableheight" cellspacing="1" border="0" cellpadding="0" style="padding-left: 2px; cursor: default;">
-			<col width="60px" />
+			<col width="120px" />
 			<col width="59px" />
-			<col width="60px" />
 
 			<tr>
-				<td style="text-align:center; vertical-align:middle;">
+				<td style="text-align:center; vertical-align:middle;" colspan="3">
 					#if $cur_airs_next
 					#set $ldatetime = $network_timezones.parse_date_time($cur_airs_next,$curShow.airs,$curShow.network)
-						<div class="show-date">Next Ep: $sbdatetime.sbdatetime.sbfdate($ldatetime)</div>
+						<div class="show-date">Next Ep: <span class="${fuzzydate}">$sbdatetime.sbdatetime.sbfdate($ldatetime)</span></div>
 					#else if $curShow.status != "Ended" and int($curShow.paused) == 1:
 						<div class="show-status">Paused</div>
 					#else if $curShow.status:
@@ -427,28 +427,29 @@ $myShowList.sort(lambda x, y: cmp(x.name, y.name))
 						<div class="show-status">?</div>
 					#end if
 				</td>
-					
+			</tr>
-				<td style="text-align:center; vertical-align:middle;">
+			<tr>
-				    #if $layout != 'simple':	
-						#if $curShow.network:
-							<img class="show-network-image" src="$sbRoot/images/network/${curShow.network.replace(u"\u00C9",'e').lower()}.png" alt="$curShow.network" title="$curShow.network" />	
-						#else:
-							<img class="show-network-image" src="$sbRoot/images/network/nonetwork.png" alt="No Network" title="No Network" />
-						#end if
-					#else:
-						$curShow.network
-					#end if	
-				</td>
-					
 				<td style="text-align:center; vertical-align:middle;">
 					<span class="show-dlstats" title="$download_stat_tip">$download_stat</span>
-					<div class="float-right">
+					<div class="float-left">
 					#if $curShow.quality in $qualityPresets:
 						<span class="show-dlstats">$qualityPresetStrings[$curShow.quality]</span>
 					#else:
 						<span class="show-dlstats">Custom</span>
 					#end if
 				</td>
+					
+				<td style="text-align:center; vertical-align:middle;">
+				    #if $layout != 'simple':
+						#if $curShow.network:
+							<img class="show-network-image float-right" src="$sbRoot/images/network/${curShow.network.replace(u"\u00C9",'e').lower()}.png" alt="$curShow.network" title="$curShow.network" />
+						#else:
+							<img class="show-network-image float-right" src="$sbRoot/images/network/nonetwork.png" alt="No Network" title="No Network" />
+						#end if
+					#else:
+						$curShow.network
+					#end if
+				</td>
 			</tr>
 		</table>
           

gui/slick/interfaces/default/home_trendingShows.tmpl

@@ -23,14 +23,15 @@
 
 <table id="trakt" width="100%" cellspacing="1" border="0" cellpadding="0">
 #for $i, $cur_show in $enumerate($trending_shows):
-#if not $i%6
+#set $image = re.sub(r"(.*)(\..*?)$", r"\1-300\2", $cur_show["images"]["poster"], 0, re.IGNORECASE | re.MULTILINE)
+#if not $i%5
 <tr>
 #end if
 
 <td class="trakt_show">
 	<div class="traktContainer">
 	<div class="trakt-image">
-    <a href="${cur_show["url"]}" target="_blank"><img alt="" class="trakt-image" src="${cur_show["images"]["poster"]}" /></a>
+    <a href="${cur_show["url"]}" target="_blank"><img alt="" class="trakt-image" src="${image}" /></a>
 		<div class="trakt-image-slide">
 			<p>$cur_show["title"]</p>
 		</div>

gui/slick/interfaces/default/manage_massEdit.tmpl

@@ -72,6 +72,23 @@
     </div>
 </div>
 
+#if $anyQualities + $bestQualities:
+#set $isSelected = ' selected="selected"'
+#set $isEnabled = $isSelected
+#set $isDisabled = $isSelected
+#if $archive_firstmatch_value##set $isDisabled = ''##else##set $isEnabled = ''##end if#
+<div class="optionWrapper clearfix">
+<span class="selectTitle">Archive on first match</span>
+    <div class="selectChoices">
+        <select id="edit_archive_firstmatch" name="archive_firstmatch">
+            <option value="keep">&lt; keep &gt;</option>
+            <option value="enable"${isEnabled}>enable</option>
+            <option value="disable"${isDisabled}>disable</option>
+        </select>
+    </div>
+</div>
+#end if
+
 <div class="optionWrapper clearfix">
 <span class="selectTitle">Flatten Folders <span class="separator">*</span></span>
     <div class="selectChoices">

gui/slick/js/fuzzyMoment.js

@@ -7,6 +7,7 @@
  * timeFormat string The python token time formatting
  * trimZero Whether to trim leading "0"s (default : false)
  * dtGlue string To insert between the output of date and time (default: '<br />')
+ * dtInline Bool Whether to output date inline or use more than one line
  */
  function fuzzyMoment(fmConfig) {
 
@@ -16,6 +17,7 @@
         timeFormat = (/undefined/i.test(typeof(fmConfig)) || /undefined/i.test(typeof(fmConfig.timeFormat)) ? '' : fmConfig.timeFormat),
         trimZero = (/undefined/i.test(typeof(fmConfig)) || /undefined/i.test(typeof(fmConfig.trimZero)) ? false : !!fmConfig.trimZero),
         dtGlue = (/undefined/i.test(typeof(fmConfig)) || /undefined/i.test(typeof(fmConfig.dtGlue)) ? '<br />' : fmConfig.dtGlue),
+        dtInline = (/undefined/i.test(typeof(fmConfig)) || /undefined/i.test(typeof(fmConfig.dtInline)) ? false : fmConfig.dtInline),
 
         jd = (function (str) {
             var token_map = ['a', 'ddd', 'A', 'dddd', 'b', 'MMM', 'B', 'MMMM', 'd', 'DD', 'm', 'MM', 'y', 'YY', 'Y', 'YYYY', 'x', 'L',
@@ -134,7 +136,7 @@
                 qTipTime = true;
                 var month = airdate.diff(today, 'month');
                 if (1 == parseInt(airdate.year() - today.year()))
-                    result += '<br />(Next Year)';
+                    result += (dtInline ? ' ' : '<br />') + '(Next Year)';
             }
             titleThis = true;
         }

gui/slick/js/submenu.js

@@ -25,8 +25,8 @@ function initActions() {
     $("#SubMenu a[href$='/home/addShows/']").addClass('btn').html('<span class="ui-icon ui-icon-video pull-left"></span> Add Show');
     $("#SubMenu a:contains('Processing')").addClass('btn').html('<span class="ui-icon ui-icon-folder-open pull-left"></span> Post-Processing');
     $("#SubMenu a:contains('Manage Searches')").addClass('btn').html('<span class="ui-icon ui-icon-search pull-left"></span> Manage Searches');
-    $("#SubMenu a:contains('Manage Torrents')").addClass('btn').html('<img width="16" height="16" alt="" src="../images/menu/bittorrent.png"> Manage Torrents');
+    $("#SubMenu a:contains('Manage Torrents')").addClass('btn').html('<img width="16" height="16" alt="" src="/images/menu/bittorrent.png"> Manage Torrents');
-    $("#SubMenu a[href$='/manage/failedDownloads/']").addClass('btn').html('<img width="16" height="16" alt="" src="../images/menu/failed_download.png"> Failed Downloads');
+    $("#SubMenu a[href$='/manage/failedDownloads/']").addClass('btn').html('<img width="16" height="16" alt="" src="/images/menu/failed_download.png"> Failed Downloads');
     $("#SubMenu a:contains('Notification')").addClass('btn').html('<span class="ui-icon ui-icon-note pull-left"></span> Notification');
     $("#SubMenu a:contains('Update show in XBMC')").addClass('btn').html('<span class="ui-icon ui-icon-refresh pull-left"></span> Update show in XBMC');
     $("#SubMenu a[href$='/home/updateXBMC/']").addClass('btn').html('<span class="ui-icon ui-icon-refresh pull-left"></span> Update XBMC');

lib/certifi/__init__.py

@@ -0,0 +1 @@
+from .core import where

lib/certifi/__main__.py

@@ -0,0 +1,2 @@
+from certifi import where
+print(where())

lib/certifi/core.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+certifi.py
+~~~~~~~~~~
+
+This module returns the installation location of cacert.pem.
+"""
+
+import os
+
+def where():
+    f = os.path.split(__file__)[0]
+
+    return os.path.join(f, 'cacert.pem')
+
+if __name__ == '__main__':
+    print(where())

lib/subliminal/core.py

@@ -32,7 +32,8 @@ __all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE',
            'create_list_tasks', 'create_download_tasks', 'consume_task', 'matching_confidence',
            'key_subtitles', 'group_by_video']
 logger = logging.getLogger("subliminal")
-SERVICES = ['opensubtitles', 'subswiki', 'subtitulos', 'thesubdb', 'addic7ed', 'tvsubtitles', 'itasa', 'usub']
+SERVICES = ['opensubtitles', 'subswiki', 'subtitulos', 'thesubdb', 'addic7ed', 'tvsubtitles', 'itasa',
+            'usub', 'subscenter']
 LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE = range(4)
 
 

lib/subliminal/services/subscenter.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# Copyright 2012 Ofir Brukner <ofirbrukner@gmail.com>
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal.  If not, see <http://www.gnu.org/licenses/>.
+import logging
+import re
+import json
+from . import ServiceBase
+from ..exceptions import ServiceError
+from ..language import language_set
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..videos import Episode, Movie
+from ..utils import to_unicode, get_keywords
+
+
+logger = logging.getLogger("subliminal")
+
+
+class Subscenter(ServiceBase):
+    server_url = 'http://subscenter.cinemast.com/he/'
+    site_url = 'http://subscenter.cinemast.com/'
+    api_based = False
+    languages = language_set(['he', 'en'])
+    videos = [Episode, Movie]
+    require_video = False
+    required_features = ['permissive']
+
+    @staticmethod
+    def slugify(string):
+        new_string = string.replace(' ', '-').replace("'", '').replace(':', '').lower()
+        # We remove multiple spaces by using this regular expression.
+        return re.sub('-+', '-', new_string)
+
+    def list_checked(self, video, languages):
+        series = None
+        season = None
+        episode = None
+        title = video.title
+        if isinstance(video, Episode):
+            series = video.series
+            season = video.season
+            episode = video.episode
+        return self.query(video.path or video.release, languages, get_keywords(video.guess), series, season,
+                          episode, title)
+
+    def query(self, filepath, languages=None, keywords=None, series=None, season=None, episode=None, title=None):
+        logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
+        # Converts the title to Subscenter format by replacing whitespaces and removing specific chars.
+        if series and season and episode:
+            # Search for a TV show.
+            kind = 'episode'
+            slugified_series = self.slugify(series)
+            url = self.server_url + 'cinemast/data/series/sb/' + slugified_series + '/' + str(season) + '/' + \
+                   str(episode) + '/'
+        elif title:
+            # Search for a movie.
+            kind = 'movie'
+            slugified_title = self.slugify(title)
+            url = self.server_url + 'cinemast/data/movie/sb/' + slugified_title + '/'
+        else:
+            raise ServiceError('One or more parameters are missing')
+        logger.debug('Searching subtitles %r', {'title': title, 'season': season, 'episode': episode})
+        response = self.session.get(url)
+        if response.status_code != 200:
+            raise ServiceError('Request failed with status code %d' % response.status_code)
+
+        subtitles = []
+        response_json = json.loads(response.content)
+        for lang, lang_json in response_json.items():
+            lang_obj = self.get_language(lang)
+            if lang_obj in self.languages and lang_obj in languages:
+                for group_data in lang_json.values():
+                    for quality in group_data.values():
+                        for sub in quality.values():
+                            release = sub.get('subtitle_version')
+                            sub_path = get_subtitle_path(filepath, lang_obj, self.config.multi)
+                            link = self.server_url + 'subtitle/download/' + lang + '/' + str(sub.get('id')) + \
+                                   '/?v=' + release + '&key=' + str(sub.get('key'))
+                            subtitles.append(ResultSubtitle(sub_path, lang_obj, self.__class__.__name__.lower(),
+                                                            link, release=to_unicode(release)))
+        return subtitles
+
+    def download(self, subtitle):
+        self.download_zip_file(subtitle.link, subtitle.path)
+        return subtitle
+
+
+Service = Subscenter

sickbeard/__init__.py

@@ -127,6 +127,7 @@ PLAY_VIDEOS = False
 
 HANDLE_REVERSE_PROXY = False
 PROXY_SETTING = None
+PROXY_INDEXERS = True
 
 LOCALHOST_IP = None
 
@@ -498,7 +499,7 @@ def initialize(consoleLogging=True):
             GUI_NAME, HOME_LAYOUT, HISTORY_LAYOUT, DISPLAY_SHOW_SPECIALS, COMING_EPS_LAYOUT, COMING_EPS_SORT, COMING_EPS_DISPLAY_PAUSED, COMING_EPS_MISSED_RANGE, FUZZY_DATING, TRIM_ZERO, DATE_PRESET, TIME_PRESET, TIME_PRESET_W_SECONDS, THEME_NAME, \
             METADATA_WDTV, METADATA_TIVO, METADATA_MEDE8ER, IGNORE_WORDS, REQUIRE_WORDS, CALENDAR_UNPROTECTED, CREATE_MISSING_SHOW_DIRS, \
             ADD_SHOWS_WO_DIR, USE_SUBTITLES, SUBTITLES_LANGUAGES, SUBTITLES_DIR, SUBTITLES_SERVICES_LIST, SUBTITLES_SERVICES_ENABLED, SUBTITLES_HISTORY, SUBTITLES_FINDER_FREQUENCY, subtitlesFinderScheduler, \
-            USE_FAILED_DOWNLOADS, DELETE_FAILED, ANON_REDIRECT, LOCALHOST_IP, TMDB_API_KEY, DEBUG, PROXY_SETTING, \
+            USE_FAILED_DOWNLOADS, DELETE_FAILED, ANON_REDIRECT, LOCALHOST_IP, TMDB_API_KEY, DEBUG, PROXY_SETTING, PROXY_INDEXERS, \
             AUTOPOSTPROCESSER_FREQUENCY, DEFAULT_AUTOPOSTPROCESSER_FREQUENCY, MIN_AUTOPOSTPROCESSER_FREQUENCY, \
             ANIME_DEFAULT, NAMING_ANIME, ANIMESUPPORT, USE_ANIDB, ANIDB_USERNAME, ANIDB_PASSWORD, ANIDB_USE_MYLIST, \
             ANIME_SPLIT_HOME, SCENE_DEFAULT, PLAY_VIDEOS, BACKLOG_DAYS
@@ -597,6 +598,7 @@ def initialize(consoleLogging=True):
 
         ANON_REDIRECT = check_setting_str(CFG, 'General', 'anon_redirect', 'http://dereferer.org/?')
         PROXY_SETTING = check_setting_str(CFG, 'General', 'proxy_setting', '')
+        PROXY_INDEXERS = bool(check_setting_str(CFG, 'General', 'proxy_indexers', 1))
         # attempt to help prevent users from breaking links by using a bad url 
         if not ANON_REDIRECT.endswith('?'):
             ANON_REDIRECT = ''
@@ -1439,6 +1441,7 @@ def save_config():
     new_config['General']['update_shows_on_start'] = int(UPDATE_SHOWS_ON_START)
     new_config['General']['sort_article'] = int(SORT_ARTICLE)
     new_config['General']['proxy_setting'] = PROXY_SETTING
+    new_config['General']['proxy_indexers'] = int(PROXY_INDEXERS)
 
     new_config['General']['use_listview'] = int(USE_LISTVIEW)
     new_config['General']['metadata_xbmc'] = METADATA_XBMC

sickbeard/clients/generic.py

@@ -198,7 +198,7 @@ class GenericClient(object):
                 logger.log(self.name + u': Unable to set priority for Torrent', logger.ERROR)
 
         except Exception, e:
-            logger.log(self.name + u': Failed Sending Torrent ', logger.ERROR)
+            logger.log(self.name + u': Failed Sending Torrent: ' + result.name + ' - ' + result.hash, logger.ERROR)
             logger.log(self.name + u': Exception raised when sending torrent: ' + ex(e), logger.DEBUG)
             return r_code
 

sickbeard/helpers.py

@@ -378,8 +378,9 @@ def hardlinkFile(srcFile, destFile):
     try:
         ek.ek(link, srcFile, destFile)
         fixSetGroupID(destFile)
-    except:
+    except Exception, e:
-        logger.log(u"Failed to create hardlink of " + srcFile + " at " + destFile + ". Copying instead", logger.ERROR)
+        logger.log(u"Failed to create hardlink of " + srcFile + " at " + destFile + ": " + ex(e) + ". Copying instead",
+                   logger.ERROR)
         copyFile(srcFile, destFile)
 
 

sickbeard/indexers/indexer_api.py

@@ -48,7 +48,7 @@ class indexerApi(object):
         if self.indexerID:
             if sickbeard.CACHE_DIR:
                 indexerConfig[self.indexerID]['api_params']['cache'] = os.path.join(sickbeard.CACHE_DIR, 'indexers', self.name)
-            if sickbeard.PROXY_SETTING:
+            if sickbeard.PROXY_SETTING and sickbeard.PROXY_INDEXERS:
                 indexerConfig[self.indexerID]['api_params']['proxy'] = sickbeard.PROXY_SETTING
 
             return indexerConfig[self.indexerID]['api_params']

sickbeard/name_parser/regexes.py

@@ -88,7 +88,7 @@ normal_regexes = [
      # Show Name - 2010-11-23 - Ep Name
      '''
      ^((?P<series_name>.+?)[. _-]+)?             # Show_Name and separator
-     (?P<air_date>(\d+[. _-]\d+[. _-]\d+)|(\d+\w+[. _-]\w+[. _-]\d+))[. _-]+
+     (?P<air_date>(\d+[. _-]\d+[. _-]\d+)|(\d+\w+[. _-]\w+[. _-]\d+))
      [. _-]*((?P<extra_info>.+?)                 # Source_Quality_Etc-
      ((?<![. _-])(?<!WEB)                        # Make sure this is really the release group
      -(?P<release_group>[^- ]+([. _-]\[.*\])?))?)?$              # Group

sickbeard/providers/iptorrents.py

@@ -174,6 +174,7 @@ class IPTorrentsProvider(generic.TorrentProvider):
                     continue
 
                 try:
+                    data = re.sub(r'<button.+?<[\/]button>', '', data, 0, re.IGNORECASE | re.MULTILINE)
                     with BS4Parser(data, features=["html5lib", "permissive"]) as html:
                         if not html:
                             logger.log(u"Invalid HTML data: " + str(data), logger.DEBUG)

sickbeard/tvcache.py

@@ -51,7 +51,7 @@ class CacheDBConnection(db.DBConnection):
                     self.action("DELETE FROM [" + providerName + "] WHERE url = ?", [cur_dupe["url"]])
 
             # add unique index to prevent further dupes from happening if one does not exist
-            self.action("CREATE UNIQUE INDEX IF NOT EXISTS idx_url ON " + providerName + " (url)")
+            self.action("CREATE UNIQUE INDEX IF NOT EXISTS idx_url ON [" + providerName + "] (url)")
 
             # add release_group column to table if missing
             if not self.hasColumn(providerName, 'release_group'):

sickbeard/versionChecker.py

@@ -378,7 +378,7 @@ class GitUpdateManager(UpdateManager):
 
             if output:
                 output = output.strip()
-            logger.log(u"git output: " + output, logger.DEBUG)
+            logger.log(u"git output: " + str(output), logger.DEBUG)
 
         except OSError:
             logger.log(u"Command " + cmd + " didn't work")
@@ -389,15 +389,15 @@ class GitUpdateManager(UpdateManager):
             exit_status = 0
 
         elif exit_status == 1:
-            logger.log(cmd + u" returned : " + output, logger.ERROR)
+            logger.log(cmd + u" returned : " + str(output), logger.ERROR)
             exit_status = 1
 
         elif exit_status == 128 or 'fatal:' in output or err:
-            logger.log(cmd + u" returned : " + output, logger.ERROR)
+            logger.log(cmd + u" returned : " + str(output), logger.ERROR)
             exit_status = 128
 
         else:
-            logger.log(cmd + u" returned : " + output + u", treat as error for now", logger.ERROR)
+            logger.log(cmd + u" returned : " + str(output) + u", treat as error for now", logger.ERROR)
             exit_status = 1
 
         return (output, err, exit_status)

sickbeard/webserve.py

@@ -948,6 +948,9 @@ class Manage(MainHandler):
             if showObj:
                 showList.append(showObj)
 
+        archive_firstmatch_all_same = True
+        last_archive_firstmatch = None
+
         flatten_folders_all_same = True
         last_flatten_folders = None
 
@@ -980,6 +983,13 @@ class Manage(MainHandler):
             if cur_root_dir not in root_dir_list:
                 root_dir_list.append(cur_root_dir)
 
+            if archive_firstmatch_all_same:
+                # if we had a value already and this value is different then they're not all the same
+                if last_archive_firstmatch not in (None, curShow.archive_firstmatch):
+                    archive_firstmatch_all_same = False
+                else:
+                    last_archive_firstmatch = curShow.archive_firstmatch
+
             # if we know they're not all the same then no point even bothering
             if paused_all_same:
                 # if we had a value already and this value is different then they're not all the same
@@ -1032,6 +1042,7 @@ class Manage(MainHandler):
                     last_air_by_date = curShow.air_by_date
 
         t.showList = toEdit
+        t.archive_firstmatch_value = last_archive_firstmatch if archive_firstmatch_all_same else None
         t.paused_value = last_paused if paused_all_same else None
         t.anime_value = last_anime if anime_all_same else None
         t.flatten_folders_value = last_flatten_folders if flatten_folders_all_same else None
@@ -1045,7 +1056,7 @@ class Manage(MainHandler):
         return _munge(t)
 
 
-    def massEditSubmit(self, paused=None, anime=None, sports=None, scene=None, flatten_folders=None,
+    def massEditSubmit(self, archive_firstmatch=None, paused=None, anime=None, sports=None, scene=None, flatten_folders=None,
                        quality_preset=False,
                        subtitles=None, air_by_date=None, anyQualities=[], bestQualities=[], toEdit=None, *args,
                        **kwargs):
@@ -1075,6 +1086,12 @@ class Manage(MainHandler):
             else:
                 new_show_dir = showObj._location
 
+            if archive_firstmatch == 'keep':
+                new_archive_firstmatch = showObj.archive_firstmatch
+            else:
+                new_archive_firstmatch = True if archive_firstmatch == 'enable' else False
+            new_archive_firstmatch = 'on' if new_archive_firstmatch else 'off'
+
             if paused == 'keep':
                 new_paused = showObj.paused
             else:
@@ -1125,6 +1142,7 @@ class Manage(MainHandler):
 
             curErrors += Home(self.application, self.request).editShow(curShow, new_show_dir, anyQualities,
                                                                        bestQualities, exceptions_list,
+                                                                       archive_firstmatch=new_archive_firstmatch,
                                                                        flatten_folders=new_flatten_folders,
                                                                        paused=new_paused, sports=new_sports,
                                                                        subtitles=new_subtitles, anime=new_anime,
@@ -1486,7 +1504,7 @@ class ConfigGeneral(MainHandler):
                     use_api=None, api_key=None, indexer_default=None, timezone_display=None, cpu_preset=None,
                     web_password=None, version_notify=None, enable_https=None, https_cert=None, https_key=None,
                     handle_reverse_proxy=None, sort_article=None, auto_update=None, notify_on_update=None,
-                    proxy_setting=None, anon_redirect=None, git_path=None, calendar_unprotected=None,
+                    proxy_setting=None, proxy_indexers=None, anon_redirect=None, git_path=None, calendar_unprotected=None,
                     fuzzy_dating=None, trim_zero=None, date_preset=None, date_preset_na=None, time_preset=None,
                     indexer_timeout=None, play_videos=None, rootDir=None, use_imdbwl=None, imdbWatchlistCsv=None, theme_name=None):
 
@@ -1508,6 +1526,7 @@ class ConfigGeneral(MainHandler):
         sickbeard.CPU_PRESET = cpu_preset
         sickbeard.ANON_REDIRECT = anon_redirect
         sickbeard.PROXY_SETTING = proxy_setting
+        sickbeard.PROXY_INDEXERS = config.checkbox_to_value(proxy_indexers)
         sickbeard.GIT_PATH = git_path
         sickbeard.CALENDAR_UNPROTECTED = config.checkbox_to_value(calendar_unprotected)
         # sickbeard.LOG_DIR is set in config.change_LOG_DIR()
@@ -3932,6 +3951,7 @@ class Home(MainHandler):
         with showObj.lock:
             newQuality = Quality.combineQualities(map(int, anyQualities), map(int, bestQualities))
             showObj.quality = newQuality
+            showObj.archive_firstmatch = archive_firstmatch
 
             # reversed for now
             if bool(showObj.flatten_folders) != bool(flatten_folders):
@@ -3951,7 +3971,6 @@ class Home(MainHandler):
             if not directCall:
                 showObj.lang = indexer_lang
                 showObj.dvdorder = dvdorder
-                showObj.archive_firstmatch = archive_firstmatch
                 showObj.rls_ignore_words = rls_ignore_words.strip()
                 showObj.rls_require_words = rls_require_words.strip()
 
@@ -4450,30 +4469,29 @@ class Home(MainHandler):
         return quality_class
 
     def searchEpisodeSubtitles(self, show=None, season=None, episode=None):
-
         # retrieve the episode object and fail if we can't get one
         ep_obj = _getEpisode(show, season, episode)
         if isinstance(ep_obj, str):
             return json.dumps({'result': 'failure'})
 
         # try do download subtitles for that episode
-        previous_subtitles = ep_obj.subtitles
+        previous_subtitles = set(subliminal.language.Language(x) for x in ep_obj.subtitles)
         try:
-            ep_obj.subtitles = ep_obj.downloadSubtitles()
+            ep_obj.subtitles = set(x.language for x in ep_obj.downloadSubtitles().values()[0])
         except:
             return json.dumps({'result': 'failure'})
 
         # return the correct json value
         if previous_subtitles != ep_obj.subtitles:
             status = 'New subtitles downloaded: %s' % ' '.join([
-                "<img src='" + sickbeard.WEB_ROOT + "/images/flags/" + subliminal.language.Language(
+                "<img src='" + sickbeard.WEB_ROOT + "/images/flags/" + x.alpha2 +
-                    x).alpha2 + ".png' alt='" + subliminal.language.Language(x).name + "'/>" for x in
+                ".png' alt='" + x.name + "'/>" for x in
-                sorted(list(set(ep_obj.subtitles).difference(previous_subtitles)))])
+                sorted(list(ep_obj.subtitles.difference(previous_subtitles)))])
         else:
             status = 'No subtitles downloaded'
         ui.notifications.message('Subtitles Search', status)
-        return json.dumps({'result': status, 'subtitles': ','.join([x for x in ep_obj.subtitles])})
+        return json.dumps({'result': status, 'subtitles': ','.join(sorted([x.alpha2 for x in
-
+                                                                    ep_obj.subtitles.union(previous_subtitles)]))})
 
     def setSceneNumbering(self, show, indexer, forSeason=None, forEpisode=None, forAbsolute=None, sceneSeason=None,
                           sceneEpisode=None, sceneAbsolute=None):

tornado/__init__.py

@@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement
 # is zero for an official release, positive for a development branch,
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
-version = "4.0b1"
+version = "4.1.dev1"
-version_info = (4, 0, 0, -99)
+version_info = (4, 1, 0, -100)

tornado/auth.py

@@ -76,7 +76,7 @@ from tornado import escape
 from tornado.httputil import url_concat
 from tornado.log import gen_log
 from tornado.stack_context import ExceptionStackContext
-from tornado.util import bytes_type, u, unicode_type, ArgReplacer
+from tornado.util import u, unicode_type, ArgReplacer
 
 try:
     import urlparse  # py2
@@ -333,7 +333,7 @@ class OAuthMixin(object):
 
         The ``callback_uri`` may be omitted if you have previously
         registered a callback URI with the third-party service.  For
-        some sevices (including Friendfeed), you must use a
+        some services (including Friendfeed), you must use a
         previously-registered callback URI and cannot specify a
         callback via this method.
 
@@ -1112,7 +1112,7 @@ class FacebookMixin(object):
             args["cancel_url"] = urlparse.urljoin(
                 self.request.full_url(), cancel_uri)
         if extended_permissions:
-            if isinstance(extended_permissions, (unicode_type, bytes_type)):
+            if isinstance(extended_permissions, (unicode_type, bytes)):
                 extended_permissions = [extended_permissions]
             args["req_perms"] = ",".join(extended_permissions)
         self.redirect("http://www.facebook.com/login.php?" +

tornado/concurrent.py

@@ -29,6 +29,7 @@ import sys
 
 from tornado.stack_context import ExceptionStackContext, wrap
 from tornado.util import raise_exc_info, ArgReplacer
+from tornado.log import app_log
 
 try:
     from concurrent import futures
@@ -173,8 +174,11 @@ class Future(object):
     def _set_done(self):
         self._done = True
         for cb in self._callbacks:
-            # TODO: error handling
+            try:
-            cb(self)
+                cb(self)
+            except Exception:
+                app_log.exception('exception calling callback %r for %r',
+                                  cb, self)
         self._callbacks = None
 
 TracebackFuture = Future

tornado/curl_httpclient.py

@@ -19,10 +19,12 @@
 from __future__ import absolute_import, division, print_function, with_statement
 
 import collections
+import functools
 import logging
 import pycurl
 import threading
 import time
+from io import BytesIO
 
 from tornado import httputil
 from tornado import ioloop
@@ -31,12 +33,6 @@ from tornado import stack_context
 
 from tornado.escape import utf8, native_str
 from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main
-from tornado.util import bytes_type
-
-try:
-    from io import BytesIO  # py3
-except ImportError:
-    from cStringIO import StringIO as BytesIO  # py2
 
 
 class CurlAsyncHTTPClient(AsyncHTTPClient):
@@ -45,7 +41,7 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
         self._multi = pycurl.CurlMulti()
         self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout)
         self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket)
-        self._curls = [_curl_create() for i in range(max_clients)]
+        self._curls = [self._curl_create() for i in range(max_clients)]
         self._free_list = self._curls[:]
         self._requests = collections.deque()
         self._fds = {}
@@ -211,8 +207,8 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
                         "callback": callback,
                         "curl_start_time": time.time(),
                     }
-                    _curl_setup_request(curl, request, curl.info["buffer"],
+                    self._curl_setup_request(curl, request, curl.info["buffer"],
-                                        curl.info["headers"])
+                                             curl.info["headers"])
                     self._multi.add_handle(curl)
 
                 if not started:
@@ -259,6 +255,206 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
     def handle_callback_exception(self, callback):
         self.io_loop.handle_callback_exception(callback)
 
+    def _curl_create(self):
+        curl = pycurl.Curl()
+        if gen_log.isEnabledFor(logging.DEBUG):
+            curl.setopt(pycurl.VERBOSE, 1)
+            curl.setopt(pycurl.DEBUGFUNCTION, self._curl_debug)
+        return curl
+
+    def _curl_setup_request(self, curl, request, buffer, headers):
+        curl.setopt(pycurl.URL, native_str(request.url))
+
+        # libcurl's magic "Expect: 100-continue" behavior causes delays
+        # with servers that don't support it (which include, among others,
+        # Google's OpenID endpoint).  Additionally, this behavior has
+        # a bug in conjunction with the curl_multi_socket_action API
+        # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976),
+        # which increases the delays.  It's more trouble than it's worth,
+        # so just turn off the feature (yes, setting Expect: to an empty
+        # value is the official way to disable this)
+        if "Expect" not in request.headers:
+            request.headers["Expect"] = ""
+
+        # libcurl adds Pragma: no-cache by default; disable that too
+        if "Pragma" not in request.headers:
+            request.headers["Pragma"] = ""
+
+        curl.setopt(pycurl.HTTPHEADER,
+                    ["%s: %s" % (native_str(k), native_str(v))
+                     for k, v in request.headers.get_all()])
+
+        curl.setopt(pycurl.HEADERFUNCTION,
+                    functools.partial(self._curl_header_callback,
+                        headers, request.header_callback))
+        if request.streaming_callback:
+            write_function = lambda chunk: self.io_loop.add_callback(
+                request.streaming_callback, chunk)
+        else:
+            write_function = buffer.write
+        if bytes is str:  # py2
+            curl.setopt(pycurl.WRITEFUNCTION, write_function)
+        else:  # py3
+            # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes
+            # a fork/port.  That version has a bug in which it passes unicode
+            # strings instead of bytes to the WRITEFUNCTION.  This means that
+            # if you use a WRITEFUNCTION (which tornado always does), you cannot
+            # download arbitrary binary data.  This needs to be fixed in the
+            # ported pycurl package, but in the meantime this lambda will
+            # make it work for downloading (utf8) text.
+            curl.setopt(pycurl.WRITEFUNCTION, lambda s: write_function(utf8(s)))
+        curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
+        curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
+        curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout))
+        curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout))
+        if request.user_agent:
+            curl.setopt(pycurl.USERAGENT, native_str(request.user_agent))
+        else:
+            curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
+        if request.network_interface:
+            curl.setopt(pycurl.INTERFACE, request.network_interface)
+        if request.decompress_response:
+            curl.setopt(pycurl.ENCODING, "gzip,deflate")
+        else:
+            curl.setopt(pycurl.ENCODING, "none")
+        if request.proxy_host and request.proxy_port:
+            curl.setopt(pycurl.PROXY, request.proxy_host)
+            curl.setopt(pycurl.PROXYPORT, request.proxy_port)
+            if request.proxy_username:
+                credentials = '%s:%s' % (request.proxy_username,
+                                         request.proxy_password)
+                curl.setopt(pycurl.PROXYUSERPWD, credentials)
+        else:
+            curl.setopt(pycurl.PROXY, '')
+            curl.unsetopt(pycurl.PROXYUSERPWD)
+        if request.validate_cert:
+            curl.setopt(pycurl.SSL_VERIFYPEER, 1)
+            curl.setopt(pycurl.SSL_VERIFYHOST, 2)
+        else:
+            curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+            curl.setopt(pycurl.SSL_VERIFYHOST, 0)
+        if request.ca_certs is not None:
+            curl.setopt(pycurl.CAINFO, request.ca_certs)
+        else:
+            # There is no way to restore pycurl.CAINFO to its default value
+            # (Using unsetopt makes it reject all certificates).
+            # I don't see any way to read the default value from python so it
+            # can be restored later.  We'll have to just leave CAINFO untouched
+            # if no ca_certs file was specified, and require that if any
+            # request uses a custom ca_certs file, they all must.
+            pass
+
+        if request.allow_ipv6 is False:
+            # Curl behaves reasonably when DNS resolution gives an ipv6 address
+            # that we can't reach, so allow ipv6 unless the user asks to disable.
+            curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
+        else:
+            curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)
+
+        # Set the request method through curl's irritating interface which makes
+        # up names for almost every single method
+        curl_options = {
+            "GET": pycurl.HTTPGET,
+            "POST": pycurl.POST,
+            "PUT": pycurl.UPLOAD,
+            "HEAD": pycurl.NOBODY,
+        }
+        custom_methods = set(["DELETE", "OPTIONS", "PATCH"])
+        for o in curl_options.values():
+            curl.setopt(o, False)
+        if request.method in curl_options:
+            curl.unsetopt(pycurl.CUSTOMREQUEST)
+            curl.setopt(curl_options[request.method], True)
+        elif request.allow_nonstandard_methods or request.method in custom_methods:
+            curl.setopt(pycurl.CUSTOMREQUEST, request.method)
+        else:
+            raise KeyError('unknown method ' + request.method)
+
+        # Handle curl's cryptic options for every individual HTTP method
+        if request.method == "GET":
+            if request.body is not None:
+                raise ValueError('Body must be None for GET request')
+        elif request.method in ("POST", "PUT") or request.body:
+            if request.body is None:
+                raise ValueError(
+                    'Body must not be None for "%s" request'
+                    % request.method)
+
+            request_buffer = BytesIO(utf8(request.body))
+            def ioctl(cmd):
+                if cmd == curl.IOCMD_RESTARTREAD:
+                    request_buffer.seek(0)
+            curl.setopt(pycurl.READFUNCTION, request_buffer.read)
+            curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
+            if request.method == "POST":
+                curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
+            else:
+                curl.setopt(pycurl.UPLOAD, True)
+                curl.setopt(pycurl.INFILESIZE, len(request.body))
+
+        if request.auth_username is not None:
+            userpwd = "%s:%s" % (request.auth_username, request.auth_password or '')
+
+            if request.auth_mode is None or request.auth_mode == "basic":
+                curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
+            elif request.auth_mode == "digest":
+                curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
+            else:
+                raise ValueError("Unsupported auth_mode %s" % request.auth_mode)
+
+            curl.setopt(pycurl.USERPWD, native_str(userpwd))
+            gen_log.debug("%s %s (username: %r)", request.method, request.url,
+                          request.auth_username)
+        else:
+            curl.unsetopt(pycurl.USERPWD)
+            gen_log.debug("%s %s", request.method, request.url)
+
+        if request.client_cert is not None:
+            curl.setopt(pycurl.SSLCERT, request.client_cert)
+
+        if request.client_key is not None:
+            curl.setopt(pycurl.SSLKEY, request.client_key)
+
+        if threading.activeCount() > 1:
+            # libcurl/pycurl is not thread-safe by default.  When multiple threads
+            # are used, signals should be disabled.  This has the side effect
+            # of disabling DNS timeouts in some environments (when libcurl is
+            # not linked against ares), so we don't do it when there is only one
+            # thread.  Applications that use many short-lived threads may need
+            # to set NOSIGNAL manually in a prepare_curl_callback since
+            # there may not be any other threads running at the time we call
+            # threading.activeCount.
+            curl.setopt(pycurl.NOSIGNAL, 1)
+        if request.prepare_curl_callback is not None:
+            request.prepare_curl_callback(curl)
+
+    def _curl_header_callback(self, headers, header_callback, header_line):
+        header_line = native_str(header_line)
+        if header_callback is not None:
+            self.io_loop.add_callback(header_callback, header_line)
+        # header_line as returned by curl includes the end-of-line characters.
+        header_line = header_line.strip()
+        if header_line.startswith("HTTP/"):
+            headers.clear()
+            try:
+                (__, __, reason) = httputil.parse_response_start_line(header_line)
+                header_line = "X-Http-Reason: %s" % reason
+            except httputil.HTTPInputError:
+                return
+        if not header_line:
+            return
+        headers.parse_line(header_line)
+
+    def _curl_debug(self, debug_type, debug_msg):
+        debug_types = ('I', '<', '>', '<', '>')
+        if debug_type == 0:
+            gen_log.debug('%s', debug_msg.strip())
+        elif debug_type in (1, 2):
+            for line in debug_msg.splitlines():
+                gen_log.debug('%s %s', debug_types[debug_type], line)
+        elif debug_type == 4:
+            gen_log.debug('%s %r', debug_types[debug_type], debug_msg)
+
 
 class CurlError(HTTPError):
     def __init__(self, errno, message):
@@ -266,212 +462,6 @@ class CurlError(HTTPError):
         self.errno = errno
 
 
-def _curl_create():
-    curl = pycurl.Curl()
-    if gen_log.isEnabledFor(logging.DEBUG):
-        curl.setopt(pycurl.VERBOSE, 1)
-        curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
-    return curl
-
-
-def _curl_setup_request(curl, request, buffer, headers):
-    curl.setopt(pycurl.URL, native_str(request.url))
-
-    # libcurl's magic "Expect: 100-continue" behavior causes delays
-    # with servers that don't support it (which include, among others,
-    # Google's OpenID endpoint).  Additionally, this behavior has
-    # a bug in conjunction with the curl_multi_socket_action API
-    # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976),
-    # which increases the delays.  It's more trouble than it's worth,
-    # so just turn off the feature (yes, setting Expect: to an empty
-    # value is the official way to disable this)
-    if "Expect" not in request.headers:
-        request.headers["Expect"] = ""
-
-    # libcurl adds Pragma: no-cache by default; disable that too
-    if "Pragma" not in request.headers:
-        request.headers["Pragma"] = ""
-
-    # Request headers may be either a regular dict or HTTPHeaders object
-    if isinstance(request.headers, httputil.HTTPHeaders):
-        curl.setopt(pycurl.HTTPHEADER,
-                    [native_str("%s: %s" % i) for i in request.headers.get_all()])
-    else:
-        curl.setopt(pycurl.HTTPHEADER,
-                    [native_str("%s: %s" % i) for i in request.headers.items()])
-
-    if request.header_callback:
-        curl.setopt(pycurl.HEADERFUNCTION,
-                    lambda line: request.header_callback(native_str(line)))
-    else:
-        curl.setopt(pycurl.HEADERFUNCTION,
-                    lambda line: _curl_header_callback(headers,
-                                                       native_str(line)))
-    if request.streaming_callback:
-        write_function = request.streaming_callback
-    else:
-        write_function = buffer.write
-    if bytes_type is str:  # py2
-        curl.setopt(pycurl.WRITEFUNCTION, write_function)
-    else:  # py3
-        # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes
-        # a fork/port.  That version has a bug in which it passes unicode
-        # strings instead of bytes to the WRITEFUNCTION.  This means that
-        # if you use a WRITEFUNCTION (which tornado always does), you cannot
-        # download arbitrary binary data.  This needs to be fixed in the
-        # ported pycurl package, but in the meantime this lambda will
-        # make it work for downloading (utf8) text.
-        curl.setopt(pycurl.WRITEFUNCTION, lambda s: write_function(utf8(s)))
-    curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
-    curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
-    curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout))
-    curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout))
-    if request.user_agent:
-        curl.setopt(pycurl.USERAGENT, native_str(request.user_agent))
-    else:
-        curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
-    if request.network_interface:
-        curl.setopt(pycurl.INTERFACE, request.network_interface)
-    if request.use_gzip:
-        curl.setopt(pycurl.ENCODING, "gzip,deflate")
-    else:
-        curl.setopt(pycurl.ENCODING, "none")
-    if request.proxy_host and request.proxy_port:
-        curl.setopt(pycurl.PROXY, request.proxy_host)
-        curl.setopt(pycurl.PROXYPORT, request.proxy_port)
-        if request.proxy_username:
-            credentials = '%s:%s' % (request.proxy_username,
-                                     request.proxy_password)
-            curl.setopt(pycurl.PROXYUSERPWD, credentials)
-    else:
-        curl.setopt(pycurl.PROXY, '')
-        curl.unsetopt(pycurl.PROXYUSERPWD)
-    if request.validate_cert:
-        curl.setopt(pycurl.SSL_VERIFYPEER, 1)
-        curl.setopt(pycurl.SSL_VERIFYHOST, 2)
-    else:
-        curl.setopt(pycurl.SSL_VERIFYPEER, 0)
-        curl.setopt(pycurl.SSL_VERIFYHOST, 0)
-    if request.ca_certs is not None:
-        curl.setopt(pycurl.CAINFO, request.ca_certs)
-    else:
-        # There is no way to restore pycurl.CAINFO to its default value
-        # (Using unsetopt makes it reject all certificates).
-        # I don't see any way to read the default value from python so it
-        # can be restored later.  We'll have to just leave CAINFO untouched
-        # if no ca_certs file was specified, and require that if any
-        # request uses a custom ca_certs file, they all must.
-        pass
-
-    if request.allow_ipv6 is False:
-        # Curl behaves reasonably when DNS resolution gives an ipv6 address
-        # that we can't reach, so allow ipv6 unless the user asks to disable.
-        curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
-    else:
-        curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)
-
-    # Set the request method through curl's irritating interface which makes
-    # up names for almost every single method
-    curl_options = {
-        "GET": pycurl.HTTPGET,
-        "POST": pycurl.POST,
-        "PUT": pycurl.UPLOAD,
-        "HEAD": pycurl.NOBODY,
-    }
-    custom_methods = set(["DELETE", "OPTIONS", "PATCH"])
-    for o in curl_options.values():
-        curl.setopt(o, False)
-    if request.method in curl_options:
-        curl.unsetopt(pycurl.CUSTOMREQUEST)
-        curl.setopt(curl_options[request.method], True)
-    elif request.allow_nonstandard_methods or request.method in custom_methods:
-        curl.setopt(pycurl.CUSTOMREQUEST, request.method)
-    else:
-        raise KeyError('unknown method ' + request.method)
-
-    # Handle curl's cryptic options for every individual HTTP method
-    if request.method in ("POST", "PUT"):
-        if request.body is None:
-            raise AssertionError(
-                'Body must not be empty for "%s" request'
-                % request.method)
-
-        request_buffer = BytesIO(utf8(request.body))
-        curl.setopt(pycurl.READFUNCTION, request_buffer.read)
-        if request.method == "POST":
-            def ioctl(cmd):
-                if cmd == curl.IOCMD_RESTARTREAD:
-                    request_buffer.seek(0)
-            curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
-            curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
-        else:
-            curl.setopt(pycurl.INFILESIZE, len(request.body))
-    elif request.method == "GET":
-        if request.body is not None:
-            raise AssertionError('Body must be empty for GET request')
-
-    if request.auth_username is not None:
-        userpwd = "%s:%s" % (request.auth_username, request.auth_password or '')
-
-        if request.auth_mode is None or request.auth_mode == "basic":
-            curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
-        elif request.auth_mode == "digest":
-            curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
-        else:
-            raise ValueError("Unsupported auth_mode %s" % request.auth_mode)
-
-        curl.setopt(pycurl.USERPWD, native_str(userpwd))
-        gen_log.debug("%s %s (username: %r)", request.method, request.url,
-                      request.auth_username)
-    else:
-        curl.unsetopt(pycurl.USERPWD)
-        gen_log.debug("%s %s", request.method, request.url)
-
-    if request.client_cert is not None:
-        curl.setopt(pycurl.SSLCERT, request.client_cert)
-
-    if request.client_key is not None:
-        curl.setopt(pycurl.SSLKEY, request.client_key)
-
-    if threading.activeCount() > 1:
-        # libcurl/pycurl is not thread-safe by default.  When multiple threads
-        # are used, signals should be disabled.  This has the side effect
-        # of disabling DNS timeouts in some environments (when libcurl is
-        # not linked against ares), so we don't do it when there is only one
-        # thread.  Applications that use many short-lived threads may need
-        # to set NOSIGNAL manually in a prepare_curl_callback since
-        # there may not be any other threads running at the time we call
-        # threading.activeCount.
-        curl.setopt(pycurl.NOSIGNAL, 1)
-    if request.prepare_curl_callback is not None:
-        request.prepare_curl_callback(curl)
-
-
-def _curl_header_callback(headers, header_line):
-    # header_line as returned by curl includes the end-of-line characters.
-    header_line = header_line.strip()
-    if header_line.startswith("HTTP/"):
-        headers.clear()
-        try:
-            (__, __, reason) = httputil.parse_response_start_line(header_line)
-            header_line = "X-Http-Reason: %s" % reason
-        except httputil.HTTPInputError:
-            return
-    if not header_line:
-        return
-    headers.parse_line(header_line)
-
-
-def _curl_debug(debug_type, debug_msg):
-    debug_types = ('I', '<', '>', '<', '>')
-    if debug_type == 0:
-        gen_log.debug('%s', debug_msg.strip())
-    elif debug_type in (1, 2):
-        for line in debug_msg.splitlines():
-            gen_log.debug('%s %s', debug_types[debug_type], line)
-    elif debug_type == 4:
-        gen_log.debug('%s %r', debug_types[debug_type], debug_msg)
-
 if __name__ == "__main__":
     AsyncHTTPClient.configure(CurlAsyncHTTPClient)
     main()

tornado/escape.py

@@ -25,7 +25,7 @@ from __future__ import absolute_import, division, print_function, with_statement
 import re
 import sys
 
-from tornado.util import bytes_type, unicode_type, basestring_type, u
+from tornado.util import unicode_type, basestring_type, u
 
 try:
     from urllib.parse import parse_qs as _parse_qs  # py3
@@ -187,7 +187,7 @@ else:
         return encoded
 
 
-_UTF8_TYPES = (bytes_type, type(None))
+_UTF8_TYPES = (bytes, type(None))
 
 
 def utf8(value):
@@ -215,7 +215,7 @@ def to_unicode(value):
     """
     if isinstance(value, _TO_UNICODE_TYPES):
         return value
-    if not isinstance(value, bytes_type):
+    if not isinstance(value, bytes):
         raise TypeError(
             "Expected bytes, unicode, or None; got %r" % type(value)
         )
@@ -246,7 +246,7 @@ def to_basestring(value):
     """
     if isinstance(value, _BASESTRING_TYPES):
         return value
-    if not isinstance(value, bytes_type):
+    if not isinstance(value, bytes):
         raise TypeError(
             "Expected bytes, unicode, or None; got %r" % type(value)
         )
@@ -264,7 +264,7 @@ def recursive_unicode(obj):
         return list(recursive_unicode(i) for i in obj)
     elif isinstance(obj, tuple):
         return tuple(recursive_unicode(i) for i in obj)
-    elif isinstance(obj, bytes_type):
+    elif isinstance(obj, bytes):
         return to_unicode(obj)
     else:
         return obj

tornado/gen.py

@@ -29,16 +29,7 @@ could be written with ``gen`` as::
 Most asynchronous functions in Tornado return a `.Future`;
 yielding this object returns its `~.Future.result`.
 
-For functions that do not return ``Futures``, `Task` works with any
+You can also yield a list or dict of ``Futures``, which will be
-function that takes a ``callback`` keyword argument (most Tornado functions
-can be used in either style, although the ``Future`` style is preferred
-since it is both shorter and provides better exception handling)::
-
-    @gen.coroutine
-    def get(self):
-        yield gen.Task(AsyncHTTPClient().fetch, "http://example.com")
-
-You can also yield a list or dict of ``Futures`` and/or ``Tasks``, which will be
 started at the same time and run in parallel; a list or dict of results will
 be returned when they are all finished::
 
@@ -54,30 +45,6 @@ be returned when they are all finished::
 
 .. versionchanged:: 3.2
    Dict support added.
-
-For more complicated interfaces, `Task` can be split into two parts:
-`Callback` and `Wait`::
-
-    class GenAsyncHandler2(RequestHandler):
-        @gen.coroutine
-        def get(self):
-            http_client = AsyncHTTPClient()
-            http_client.fetch("http://example.com",
-                              callback=(yield gen.Callback("key")))
-            response = yield gen.Wait("key")
-            do_something_with_response(response)
-            self.render("template.html")
-
-The ``key`` argument to `Callback` and `Wait` allows for multiple
-asynchronous operations to be started at different times and proceed
-in parallel: yield several callbacks with different keys, then wait
-for them once all the async operations have started.
-
-The result of a `Wait` or `Task` yield expression depends on how the callback
-was run.  If it was called with no arguments, the result is ``None``.  If
-it was called with one argument, the result is that argument.  If it was
-called with more than one argument or any keyword arguments, the result
-is an `Arguments` object, which is a named tuple ``(args, kwargs)``.
 """
 from __future__ import absolute_import, division, print_function, with_statement
 
@@ -142,7 +109,10 @@ def engine(func):
                 raise ReturnValueIgnoredError(
                     "@gen.engine functions cannot return values: %r" %
                     (future.result(),))
-        future.add_done_callback(final_callback)
+        # The engine interface doesn't give us any way to return
+        # errors but to raise them into the stack context.
+        # Save the stack context here to use when the Future has resolved.
+        future.add_done_callback(stack_context.wrap(final_callback))
     return wrapper
 
 
@@ -169,6 +139,17 @@ def coroutine(func, replace_callback=True):
 
     From the caller's perspective, ``@gen.coroutine`` is similar to
     the combination of ``@return_future`` and ``@gen.engine``.
+
+    .. warning::
+
+       When exceptions occur inside a coroutine, the exception
+       information will be stored in the `.Future` object. You must
+       examine the result of the `.Future` object, or the exception
+       may go unnoticed by your code. This means yielding the function
+       if called from another coroutine, using something like
+       `.IOLoop.run_sync` for top-level calls, or passing the `.Future`
+       to `.IOLoop.add_future`.
+
     """
     return _make_coroutine_wrapper(func, replace_callback=True)
 
@@ -218,7 +199,18 @@ def _make_coroutine_wrapper(func, replace_callback):
                     future.set_exc_info(sys.exc_info())
                 else:
                     Runner(result, future, yielded)
-                return future
+                try:
+                    return future
+                finally:
+                    # Subtle memory optimization: if next() raised an exception,
+                    # the future's exc_info contains a traceback which
+                    # includes this stack frame.  This creates a cycle,
+                    # which will be collected at the next full GC but has
+                    # been shown to greatly increase memory usage of
+                    # benchmarks (relative to the refcount-based scheme
+                    # used in the absence of cycles).  We can avoid the
+                    # cycle by clearing the local variable after we return it.
+                    future = None
         future.set_result(result)
         return future
     return wrapper
@@ -252,8 +244,8 @@ class Return(Exception):
 class YieldPoint(object):
     """Base class for objects that may be yielded from the generator.
 
-    Applications do not normally need to use this class, but it may be
+    .. deprecated:: 4.0
-    subclassed to provide additional yielding behavior.
+       Use `Futures <.Future>` instead.
     """
     def start(self, runner):
         """Called by the runner after the generator has yielded.
@@ -289,6 +281,9 @@ class Callback(YieldPoint):
 
     The callback may be called with zero or one arguments; if an argument
     is given it will be returned by `Wait`.
+
+    .. deprecated:: 4.0
+       Use `Futures <.Future>` instead.
     """
     def __init__(self, key):
         self.key = key
@@ -305,7 +300,11 @@ class Callback(YieldPoint):
 
 
 class Wait(YieldPoint):
-    """Returns the argument passed to the result of a previous `Callback`."""
+    """Returns the argument passed to the result of a previous `Callback`.
+
+    .. deprecated:: 4.0
+       Use `Futures <.Future>` instead.
+    """
     def __init__(self, key):
         self.key = key
 
@@ -326,6 +325,9 @@ class WaitAll(YieldPoint):
     a list of results in the same order.
 
     `WaitAll` is equivalent to yielding a list of `Wait` objects.
+
+    .. deprecated:: 4.0
+       Use `Futures <.Future>` instead.
     """
     def __init__(self, keys):
         self.keys = keys
@@ -341,20 +343,12 @@ class WaitAll(YieldPoint):
 
 
 def Task(func, *args, **kwargs):
-    """Runs a single asynchronous operation.
+    """Adapts a callback-based asynchronous function for use in coroutines.
 
     Takes a function (and optional additional arguments) and runs it with
     those arguments plus a ``callback`` keyword argument.  The argument passed
     to the callback is returned as the result of the yield expression.
 
-    A `Task` is equivalent to a `Callback`/`Wait` pair (with a unique
-    key generated automatically)::
-
-        result = yield gen.Task(func, args)
-
-        func(args, callback=(yield gen.Callback(key)))
-        result = yield gen.Wait(key)
-
     .. versionchanged:: 4.0
        ``gen.Task`` is now a function that returns a `.Future`, instead of
        a subclass of `YieldPoint`.  It still behaves the same way when

tornado/http1connection.py

@@ -21,6 +21,8 @@
 
 from __future__ import absolute_import, division, print_function, with_statement
 
+import re
+
 from tornado.concurrent import Future
 from tornado.escape import native_str, utf8
 from tornado import gen
@@ -56,7 +58,7 @@ class HTTP1ConnectionParameters(object):
     """
     def __init__(self, no_keep_alive=False, chunk_size=None,
                  max_header_size=None, header_timeout=None, max_body_size=None,
-                 body_timeout=None, use_gzip=False):
+                 body_timeout=None, decompress=False):
         """
         :arg bool no_keep_alive: If true, always close the connection after
             one request.
@@ -65,7 +67,8 @@ class HTTP1ConnectionParameters(object):
         :arg float header_timeout: how long to wait for all headers (seconds)
         :arg int max_body_size: maximum amount of data for body
         :arg float body_timeout: how long to wait while reading body (seconds)
-        :arg bool use_gzip: if true, decode incoming ``Content-Encoding: gzip``
+        :arg bool decompress: if true, decode incoming
+            ``Content-Encoding: gzip``
         """
         self.no_keep_alive = no_keep_alive
         self.chunk_size = chunk_size or 65536
@@ -73,7 +76,7 @@ class HTTP1ConnectionParameters(object):
         self.header_timeout = header_timeout
         self.max_body_size = max_body_size
         self.body_timeout = body_timeout
-        self.use_gzip = use_gzip
+        self.decompress = decompress
 
 
 class HTTP1Connection(httputil.HTTPConnection):
@@ -141,7 +144,7 @@ class HTTP1Connection(httputil.HTTPConnection):
         Returns a `.Future` that resolves to None after the full response has
         been read.
         """
-        if self.params.use_gzip:
+        if self.params.decompress:
             delegate = _GzipMessageDelegate(delegate, self.params.chunk_size)
         return self._read_message(delegate)
 
@@ -190,8 +193,17 @@ class HTTP1Connection(httputil.HTTPConnection):
                     skip_body = True
                 code = start_line.code
                 if code == 304:
+                    # 304 responses may include the content-length header
+                    # but do not actually have a body.
+                    # http://tools.ietf.org/html/rfc7230#section-3.3
                     skip_body = True
                 if code >= 100 and code < 200:
+                    # 1xx responses should never indicate the presence of
+                    # a body.
+                    if ('Content-Length' in headers or
+                        'Transfer-Encoding' in headers):
+                        raise httputil.HTTPInputError(
+                            "Response code %d cannot have body" % code)
                     # TODO: client delegates will get headers_received twice
                     # in the case of a 100-continue.  Document or change?
                     yield self._read_message(delegate)
@@ -200,7 +212,8 @@ class HTTP1Connection(httputil.HTTPConnection):
                         not self._write_finished):
                     self.stream.write(b"HTTP/1.1 100 (Continue)\r\n\r\n")
             if not skip_body:
-                body_future = self._read_body(headers, delegate)
+                body_future = self._read_body(
+                    start_line.code if self.is_client else 0, headers, delegate)
                 if body_future is not None:
                     if self._body_timeout is None:
                         yield body_future
@@ -293,6 +306,8 @@ class HTTP1Connection(httputil.HTTPConnection):
         self._clear_callbacks()
         stream = self.stream
         self.stream = None
+        if not self._finish_future.done():
+            self._finish_future.set_result(None)
         return stream
 
     def set_body_timeout(self, timeout):
@@ -454,6 +469,7 @@ class HTTP1Connection(httputil.HTTPConnection):
         if start_line.version == "HTTP/1.1":
             return connection_header != "close"
         elif ("Content-Length" in headers
+              or headers.get("Transfer-Encoding", "").lower() == "chunked"
               or start_line.method in ("HEAD", "GET")):
             return connection_header == "keep-alive"
         return False
@@ -470,7 +486,11 @@ class HTTP1Connection(httputil.HTTPConnection):
             self._finish_future.set_result(None)
 
     def _parse_headers(self, data):
-        data = native_str(data.decode('latin1'))
+        # The lstrip removes newlines that some implementations sometimes
+        # insert between messages of a reused connection.  Per RFC 7230,
+        # we SHOULD ignore at least one empty line before the request.
+        # http://tools.ietf.org/html/rfc7230#section-3.5
+        data = native_str(data.decode('latin1')).lstrip("\r\n")
         eol = data.find("\r\n")
         start_line = data[:eol]
         try:
@@ -481,12 +501,36 @@ class HTTP1Connection(httputil.HTTPConnection):
                                           data[eol:100])
         return start_line, headers
 
-    def _read_body(self, headers, delegate):
+    def _read_body(self, code, headers, delegate):
-        content_length = headers.get("Content-Length")
+        if "Content-Length" in headers:
-        if content_length:
+            if "," in headers["Content-Length"]:
-            content_length = int(content_length)
+                # Proxies sometimes cause Content-Length headers to get
+                # duplicated.  If all the values are identical then we can
+                # use them but if they differ it's an error.
+                pieces = re.split(r',\s*', headers["Content-Length"])
+                if any(i != pieces[0] for i in pieces):
+                    raise httputil.HTTPInputError(
+                        "Multiple unequal Content-Lengths: %r" %
+                        headers["Content-Length"])
+                headers["Content-Length"] = pieces[0]
+            content_length = int(headers["Content-Length"])
+
             if content_length > self._max_body_size:
                 raise httputil.HTTPInputError("Content-Length too long")
+        else:
+            content_length = None
+
+        if code == 204:
+            # This response code is not allowed to have a non-empty body,
+            # and has an implicit length of zero instead of read-until-close.
+            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
+            if ("Transfer-Encoding" in headers or
+                    content_length not in (None, 0)):
+                raise httputil.HTTPInputError(
+                    "Response with code %d should not have body" % code)
+            content_length = 0
+
+        if content_length is not None:
             return self._read_fixed_body(content_length, delegate)
         if headers.get("Transfer-Encoding") == "chunked":
             return self._read_chunked_body(delegate)
@@ -581,6 +625,9 @@ class _GzipMessageDelegate(httputil.HTTPMessageDelegate):
                 self._delegate.data_received(tail)
         return self._delegate.finish()
 
+    def on_connection_close(self):
+        return self._delegate.on_connection_close()
+
 
 class HTTP1ServerConnection(object):
     """An HTTP/1.x server."""

tornado/httpclient.py

@@ -33,6 +33,9 @@ information, see
 http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUTMS
 and comments in curl_httpclient.py).
 
+To select ``curl_httpclient``, call `AsyncHTTPClient.configure` at startup::
+
+    AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
@@ -60,7 +63,12 @@ class HTTPClient(object):
             response = http_client.fetch("http://www.google.com/")
             print response.body
         except httpclient.HTTPError as e:
-            print "Error:", e
+            # HTTPError is raised for non-200 responses; the response
+            # can be found in e.response.
+            print("Error: " + str(e))
+        except Exception as e:
+            # Other errors are possible, such as IOError.
+            print("Error: " + str(e))
         http_client.close()
     """
     def __init__(self, async_client_class=None, **kwargs):
@@ -279,7 +287,7 @@ class HTTPRequest(object):
         request_timeout=20.0,
         follow_redirects=True,
         max_redirects=5,
-        use_gzip=True,
+        decompress_response=True,
         proxy_password='',
         allow_nonstandard_methods=False,
         validate_cert=True)
@@ -296,7 +304,7 @@ class HTTPRequest(object):
                  validate_cert=None, ca_certs=None,
                  allow_ipv6=None,
                  client_key=None, client_cert=None, body_producer=None,
-                 expect_100_continue=False):
+                 expect_100_continue=False, decompress_response=None):
         r"""All parameters except ``url`` are optional.
 
         :arg string url: URL to fetch
@@ -330,7 +338,11 @@ class HTTPRequest(object):
            or return the 3xx response?
         :arg int max_redirects: Limit for ``follow_redirects``
         :arg string user_agent: String to send as ``User-Agent`` header
-        :arg bool use_gzip: Request gzip encoding from the server
+        :arg bool decompress_response: Request a compressed response from
+           the server and decompress it after downloading.  Default is True.
+           New in Tornado 4.0.
+        :arg bool use_gzip: Deprecated alias for ``decompress_response``
+           since Tornado 4.0.
         :arg string network_interface: Network interface to use for request.
            ``curl_httpclient`` only; see note below.
         :arg callable streaming_callback: If set, ``streaming_callback`` will
@@ -373,7 +385,6 @@ class HTTPRequest(object):
            before sending the request body.  Only supported with
            simple_httpclient.
 
-
         .. note::
 
             When using ``curl_httpclient`` certain options may be
@@ -414,7 +425,10 @@ class HTTPRequest(object):
         self.follow_redirects = follow_redirects
         self.max_redirects = max_redirects
         self.user_agent = user_agent
-        self.use_gzip = use_gzip
+        if decompress_response is not None:
+            self.decompress_response = decompress_response
+        else:
+            self.decompress_response = use_gzip
         self.network_interface = network_interface
         self.streaming_callback = streaming_callback
         self.header_callback = header_callback

tornado/httpserver.py

@@ -50,6 +50,7 @@ class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate):
 
         import tornado.httpserver
         import tornado.ioloop
+        from tornado import httputil
 
         def handle_request(request):
            message = "You requested %s\n" % request.uri
@@ -129,13 +130,14 @@ class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate):
        way other than `tornado.netutil.bind_sockets`.
 
     .. versionchanged:: 4.0
-       Added ``gzip``, ``chunk_size``, ``max_header_size``,
+       Added ``decompress_request``, ``chunk_size``, ``max_header_size``,
        ``idle_connection_timeout``, ``body_timeout``, ``max_body_size``
        arguments.  Added support for `.HTTPServerConnectionDelegate`
        instances as ``request_callback``.
     """
     def __init__(self, request_callback, no_keep_alive=False, io_loop=None,
-                 xheaders=False, ssl_options=None, protocol=None, gzip=False,
+                 xheaders=False, ssl_options=None, protocol=None,
+                 decompress_request=False,
                  chunk_size=None, max_header_size=None,
                  idle_connection_timeout=None, body_timeout=None,
                  max_body_size=None, max_buffer_size=None):
@@ -144,7 +146,7 @@ class HTTPServer(TCPServer, httputil.HTTPServerConnectionDelegate):
         self.xheaders = xheaders
         self.protocol = protocol
         self.conn_params = HTTP1ConnectionParameters(
-            use_gzip=gzip,
+            decompress=decompress_request,
             chunk_size=chunk_size,
             max_header_size=max_header_size,
             header_timeout=idle_connection_timeout or 3600,

tornado/httputil.py

@@ -33,7 +33,7 @@ import time
 
 from tornado.escape import native_str, parse_qs_bytes, utf8
 from tornado.log import gen_log
-from tornado.util import ObjectDict, bytes_type
+from tornado.util import ObjectDict
 
 try:
     import Cookie  # py2
@@ -335,7 +335,7 @@ class HTTPServerRequest(object):
 
         # set remote IP and protocol
         context = getattr(connection, 'context', None)
-        self.remote_ip = getattr(context, 'remote_ip')
+        self.remote_ip = getattr(context, 'remote_ip', None)
         self.protocol = getattr(context, 'protocol', "http")
 
         self.host = host or self.headers.get("Host") or "127.0.0.1"
@@ -379,7 +379,7 @@ class HTTPServerRequest(object):
            Use ``request.connection`` and the `.HTTPConnection` methods
            to write the response.
         """
-        assert isinstance(chunk, bytes_type)
+        assert isinstance(chunk, bytes)
         self.connection.write(chunk, callback=callback)
 
     def finish(self):
@@ -562,11 +562,18 @@ class HTTPConnection(object):
 
 
 def url_concat(url, args):
-    """Concatenate url and argument dictionary regardless of whether
+    """Concatenate url and arguments regardless of whether
     url has existing query parameters.
 
+    ``args`` may be either a dictionary or a list of key-value pairs
+    (the latter allows for multiple values with the same key.
+
+    >>> url_concat("http://example.com/foo", dict(c="d"))
+    'http://example.com/foo?c=d'
     >>> url_concat("http://example.com/foo?a=b", dict(c="d"))
     'http://example.com/foo?a=b&c=d'
+    >>> url_concat("http://example.com/foo?a=b", [("c", "d"), ("c", "d2")])
+    'http://example.com/foo?a=b&c=d&c=d2'
     """
     if not args:
         return url
@@ -803,6 +810,8 @@ def parse_response_start_line(line):
 # _parseparam and _parse_header are copied and modified from python2.7's cgi.py
 # The original 2.7 version of this code did not correctly support some
 # combinations of semicolons and double quotes.
+# It has also been modified to support valueless parameters as seen in
+# websocket extension negotiations.
 
 
 def _parseparam(s):
@@ -836,9 +845,31 @@ def _parse_header(line):
                 value = value[1:-1]
                 value = value.replace('\\\\', '\\').replace('\\"', '"')
             pdict[name] = value
+        else:
+            pdict[p] = None
     return key, pdict
 
 
+def _encode_header(key, pdict):
+    """Inverse of _parse_header.
+
+    >>> _encode_header('permessage-deflate',
+    ...     {'client_max_window_bits': 15, 'client_no_context_takeover': None})
+    'permessage-deflate; client_max_window_bits=15; client_no_context_takeover'
+    """
+    if not pdict:
+        return key
+    out = [key]
+    # Sort the parameters just to make it easy to test.
+    for k, v in sorted(pdict.items()):
+        if v is None:
+            out.append(k)
+        else:
+            # TODO: quote if necessary.
+            out.append('%s=%s' % (k, v))
+    return '; '.join(out)
+
+
 def doctests():
     import doctest
     return doctest.DocTestSuite()

tornado/ioloop.py

@@ -197,7 +197,7 @@ class IOLoop(Configurable):
 
         An `IOLoop` automatically becomes current for its thread
         when it is started, but it is sometimes useful to call
-        `make_current` explictly before starting the `IOLoop`,
+        `make_current` explicitly before starting the `IOLoop`,
         so that code run at startup time can find the right
         instance.
         """
@@ -477,7 +477,7 @@ class IOLoop(Configurable):
 
         .. versionadded:: 4.0
         """
-        self.call_at(self.time() + delay, callback, *args, **kwargs)
+        return self.call_at(self.time() + delay, callback, *args, **kwargs)
 
     def call_at(self, when, callback, *args, **kwargs):
         """Runs the ``callback`` at the absolute time designated by ``when``.
@@ -493,7 +493,7 @@ class IOLoop(Configurable):
 
         .. versionadded:: 4.0
         """
-        self.add_timeout(when, callback, *args, **kwargs)
+        return self.add_timeout(when, callback, *args, **kwargs)
 
     def remove_timeout(self, timeout):
         """Cancels a pending timeout.
@@ -724,7 +724,7 @@ class PollIOLoop(IOLoop):
         #
         # If someone has already set a wakeup fd, we don't want to
         # disturb it.  This is an issue for twisted, which does its
-        # SIGCHILD processing in response to its own wakeup fd being
+        # SIGCHLD processing in response to its own wakeup fd being
         # written to.  As long as the wakeup fd is registered on the IOLoop,
         # the loop will still wake up and everything should work.
         old_wakeup_fd = None
@@ -754,17 +754,18 @@ class PollIOLoop(IOLoop):
                 # Do not run anything until we have determined which ones
                 # are ready, so timeouts that call add_timeout cannot
                 # schedule anything in this iteration.
+                due_timeouts = []
                 if self._timeouts:
                     now = self.time()
                     while self._timeouts:
                         if self._timeouts[0].callback is None:
-                            # the timeout was cancelled
+                            # The timeout was cancelled.  Note that the
+                            # cancellation check is repeated below for timeouts
+                            # that are cancelled by another timeout or callback.
                             heapq.heappop(self._timeouts)
                             self._cancellations -= 1
                         elif self._timeouts[0].deadline <= now:
-                            timeout = heapq.heappop(self._timeouts)
+                            due_timeouts.append(heapq.heappop(self._timeouts))
-                            callbacks.append(timeout.callback)
-                            del timeout
                         else:
                             break
                     if (self._cancellations > 512
@@ -778,9 +779,12 @@ class PollIOLoop(IOLoop):
 
                 for callback in callbacks:
                     self._run_callback(callback)
+                for timeout in due_timeouts:
+                    if timeout.callback is not None:
+                        self._run_callback(timeout.callback)
                 # Closures may be holding on to a lot of memory, so allow
                 # them to be freed before we go into our poll wait.
-                callbacks = callback = None
+                callbacks = callback = due_timeouts = timeout = None
 
                 if self._callbacks:
                     # If any callbacks or timeouts called add_callback,
@@ -969,10 +973,11 @@ class PeriodicCallback(object):
         if not self._running:
             return
         try:
-            self.callback()
+            return self.callback()
         except Exception:
             self.io_loop.handle_callback_exception(self.callback)
-        self._schedule_next()
+        finally:
+            self._schedule_next()
 
     def _schedule_next(self):
         if self._running:

tornado/iostream.py

@@ -39,7 +39,7 @@ from tornado import ioloop
 from tornado.log import gen_log, app_log
 from tornado.netutil import ssl_wrap_socket, ssl_match_hostname, SSLCertificateError
 from tornado import stack_context
-from tornado.util import bytes_type, errno_from_exception
+from tornado.util import errno_from_exception
 
 try:
     from tornado.platform.posix import _set_nonblocking
@@ -324,7 +324,7 @@ class BaseIOStream(object):
         .. versionchanged:: 4.0
             Now returns a `.Future` if no callback is given.
         """
-        assert isinstance(data, bytes_type)
+        assert isinstance(data, bytes)
         self._check_closed()
         # We use bool(_write_buffer) as a proxy for write_buffer_size>0,
         # so never put empty strings in the buffer.
@@ -505,7 +505,7 @@ class BaseIOStream(object):
         def wrapper():
             self._pending_callbacks -= 1
             try:
-                callback(*args)
+                return callback(*args)
             except Exception:
                 app_log.error("Uncaught exception, closing connection.",
                               exc_info=True)
@@ -517,7 +517,8 @@ class BaseIOStream(object):
                 # Re-raise the exception so that IOLoop.handle_callback_exception
                 # can see it and log the error
                 raise
-            self._maybe_add_error_listener()
+            finally:
+                self._maybe_add_error_listener()
         # We schedule callbacks to be run on the next IOLoop iteration
         # rather than running them directly for several reasons:
         # * Prevents unbounded stack growth when a callback calls an
@@ -553,7 +554,7 @@ class BaseIOStream(object):
             # Pretend to have a pending callback so that an EOF in
             # _read_to_buffer doesn't trigger an immediate close
             # callback.  At the end of this method we'll either
-            # estabilsh a real pending callback via
+            # establish a real pending callback via
             # _read_from_buffer or run the close callback.
             #
             # We need two try statements here so that
@@ -992,6 +993,11 @@ class IOStream(BaseIOStream):
 
         """
         self._connecting = True
+        if callback is not None:
+            self._connect_callback = stack_context.wrap(callback)
+            future = None
+        else:
+            future = self._connect_future = TracebackFuture()
         try:
             self.socket.connect(address)
         except socket.error as e:
@@ -1007,12 +1013,7 @@ class IOStream(BaseIOStream):
                 gen_log.warning("Connect error on fd %s: %s",
                                 self.socket.fileno(), e)
                 self.close(exc_info=True)
-                return
+                return future
-        if callback is not None:
-            self._connect_callback = stack_context.wrap(callback)
-            future = None
-        else:
-            future = self._connect_future = TracebackFuture()
         self._add_io_state(self.io_loop.WRITE)
         return future
 
@@ -1184,8 +1185,14 @@ class SSLIOStream(IOStream):
                 return self.close(exc_info=True)
             raise
         except socket.error as err:
-            if err.args[0] in _ERRNO_CONNRESET:
+            # Some port scans (e.g. nmap in -sT mode) have been known
+            # to cause do_handshake to raise EBADF, so make that error
+            # quiet as well.
+            # https://groups.google.com/forum/?fromgroups#!topic/python-tornado/ApucKJat1_0
+            if (err.args[0] in _ERRNO_CONNRESET or
+                err.args[0] == errno.EBADF):
                 return self.close(exc_info=True)
+            raise
         except AttributeError:
             # On Linux, if the connection was reset before the call to
             # wrap_socket, do_handshake will fail with an

tornado/log.py

@@ -179,7 +179,7 @@ class LogFormatter(logging.Formatter):
 def enable_pretty_logging(options=None, logger=None):
     """Turns on formatted logging output as configured.
 
-    This is called automaticaly by `tornado.options.parse_command_line`
+    This is called automatically by `tornado.options.parse_command_line`
     and `tornado.options.parse_config_file`.
     """
     if options is None:

tornado/netutil.py

@@ -35,6 +35,11 @@ except ImportError:
     # ssl is not available on Google App Engine
     ssl = None
 
+try:
+    xrange  # py2
+except NameError:
+    xrange = range  # py3
+
 if hasattr(ssl, 'match_hostname') and hasattr(ssl, 'CertificateError'):  # python 3.2+
     ssl_match_hostname = ssl.match_hostname
     SSLCertificateError = ssl.CertificateError
@@ -60,8 +65,11 @@ _ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN)
 if hasattr(errno, "WSAEWOULDBLOCK"):
     _ERRNO_WOULDBLOCK += (errno.WSAEWOULDBLOCK,)
 
+# Default backlog used when calling sock.listen()
+_DEFAULT_BACKLOG = 128
 
-def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags=None):
+def bind_sockets(port, address=None, family=socket.AF_UNSPEC,
+                 backlog=_DEFAULT_BACKLOG, flags=None):
     """Creates listening sockets bound to the given port and address.
 
     Returns a list of socket objects (multiple sockets are returned if
@@ -141,7 +149,7 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags
     return sockets
 
 if hasattr(socket, 'AF_UNIX'):
-    def bind_unix_socket(file, mode=0o600, backlog=128):
+    def bind_unix_socket(file, mode=0o600, backlog=_DEFAULT_BACKLOG):
         """Creates a listening unix socket.
 
         If a socket with the given name already exists, it will be deleted.
@@ -184,7 +192,18 @@ def add_accept_handler(sock, callback, io_loop=None):
         io_loop = IOLoop.current()
 
     def accept_handler(fd, events):
-        while True:
+        # More connections may come in while we're handling callbacks;
+        # to prevent starvation of other tasks we must limit the number
+        # of connections we accept at a time.  Ideally we would accept
+        # up to the number of connections that were waiting when we
+        # entered this method, but this information is not available
+        # (and rearranging this method to call accept() as many times
+        # as possible before running any callbacks would have adverse
+        # effects on load balancing in multiprocess configurations).
+        # Instead, we use the (default) listen backlog as a rough
+        # heuristic for the number of connections we can reasonably
+        # accept at once.
+        for i in xrange(_DEFAULT_BACKLOG):
             try:
                 connection, address = sock.accept()
             except socket.error as e:

tornado/options.py

@@ -79,7 +79,7 @@ import sys
 import os
 import textwrap
 
-from tornado.escape import _unicode
+from tornado.escape import _unicode, native_str
 from tornado.log import define_logging_options
 from tornado import stack_context
 from tornado.util import basestring_type, exec_in
@@ -271,10 +271,14 @@ class OptionParser(object):
         If ``final`` is ``False``, parse callbacks will not be run.
         This is useful for applications that wish to combine configurations
         from multiple sources.
+
+        .. versionchanged:: 4.1
+           Config files are now always interpreted as utf-8 instead of
+           the system default encoding.
         """
         config = {}
-        with open(path) as f:
+        with open(path, 'rb') as f:
-            exec_in(f.read(), config, config)
+            exec_in(native_str(f.read()), config, config)
         for name in config:
             if name in self._options:
                 self._options[name].set(config[name])

tornado/platform/asyncio.py

@@ -10,12 +10,10 @@ unfinished callbacks on the event loop that fail when it resumes)
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
-import datetime
 import functools
 
 from tornado.ioloop import IOLoop
 from tornado import stack_context
-from tornado.util import timedelta_to_seconds
 
 try:
     # Import the real asyncio module for py33+ first.  Older versions of the

tornado/platform/twisted.py

@@ -141,7 +141,7 @@ class TornadoDelayedCall(object):
 class TornadoReactor(PosixReactorBase):
     """Twisted reactor built on the Tornado IOLoop.
 
-    Since it is intented to be used in applications where the top-level
+    Since it is intended to be used in applications where the top-level
     event loop is ``io_loop.start()`` rather than ``reactor.run()``,
     it is implemented a little differently than other Twisted reactors.
     We override `mainLoop` instead of `doIteration` and must implement

tornado/process.py

@@ -39,7 +39,7 @@ from tornado.util import errno_from_exception
 try:
     import multiprocessing
 except ImportError:
-    # Multiprocessing is not availble on Google App Engine.
+    # Multiprocessing is not available on Google App Engine.
     multiprocessing = None
 
 try:
@@ -240,7 +240,7 @@ class Subprocess(object):
 
         The callback takes one argument, the return code of the process.
 
-        This method uses a ``SIGCHILD`` handler, which is a global setting
+        This method uses a ``SIGCHLD`` handler, which is a global setting
         and may conflict if you have other libraries trying to handle the
         same signal.  If you are using more than one ``IOLoop`` it may
         be necessary to call `Subprocess.initialize` first to designate
@@ -257,7 +257,7 @@ class Subprocess(object):
 
     @classmethod
     def initialize(cls, io_loop=None):
-        """Initializes the ``SIGCHILD`` handler.
+        """Initializes the ``SIGCHLD`` handler.
 
         The signal handler is run on an `.IOLoop` to avoid locking issues.
         Note that the `.IOLoop` used for signal handling need not be the
@@ -275,7 +275,7 @@ class Subprocess(object):
 
     @classmethod
     def uninitialize(cls):
-        """Removes the ``SIGCHILD`` handler."""
+        """Removes the ``SIGCHLD`` handler."""
         if not cls._initialized:
             return
         signal.signal(signal.SIGCHLD, cls._old_sigchld)

tornado/routing.py

@@ -1,77 +0,0 @@
-'''
-Created on May 31, 2014
-
-@author: Fenriswolf
-'''
-import logging
-import inspect
-import re
-from collections import OrderedDict
-
-from tornado.web import Application, RequestHandler, HTTPError
-
-app_routing_log = logging.getLogger("tornado.application.routing")
-
-class RoutingApplication(Application):
-    def __init__(self, handlers=None, default_host="", transforms=None, wsgi=False, **settings):
-        Application.__init__(self, handlers, default_host, transforms, wsgi, **settings)
-        self.handler_map = OrderedDict()
-
-    def expose(self, rule='', methods = ['GET'], kwargs=None, name=None):
-        """
-        A decorator that is used to register a given URL rule.
-        """
-        def decorator(func, *args, **kwargs):
-            func_name = func.__name__
-            frm = inspect.stack()[1]
-            class_name = frm[3]
-            module_name = frm[0].f_back.f_globals["__name__"]
-            full_class_name = module_name + '.' + class_name
-
-            for method in methods:
-                func_rule = rule if rule else None
-                if not func_rule:
-                    if func_name == 'index':
-                        func_rule = class_name
-                    else:
-                        func_rule = class_name + '/' + func_name
-                    func_rule = r'/%s(.*)(/?)' % func_rule
-
-                if full_class_name not in self.handler_map:
-                    self.handler_map.setdefault(full_class_name, {})[method] = [(func_rule, func_name)]
-                else:
-                    self.handler_map[full_class_name][method] += [(func_rule, func_name)]
-
-                app_routing_log.info("register %s %s to %s.%s" % (method, func_rule, full_class_name, func_name))
-
-            return func
-        return decorator
-
-    def setRouteHandlers(self):
-        handlers = [(rule[0], full_class_name)
-                    for full_class_name, methods in self.handler_map.items()
-                    for rules in methods.values()
-                    for rule in rules]
-        self.add_handlers(".*$", handlers)
-
-class RequestRoutingHandler(RequestHandler):
-    def _get_func_name(self):
-        full_class_name = self.__module__ + '.' + self.__class__.__name__
-        rules = self.application.handler_map.get(full_class_name, {}).get(self.request.method, [])
-
-        for rule, func_name in rules:
-            if not rule or not func_name:
-                continue
-
-            match = re.match(rule, self.request.path)
-            if match:
-                return func_name
-
-        raise HTTPError(404, "")
-
-    def _execute_method(self):
-        if not self._finished:
-            func_name = self._get_func_name()
-            method = getattr(self, func_name)
-            self._when_complete(method(*self.path_args, **self.path_kwargs),
-                                self._execute_finish)

tornado/simple_httpclient.py

@@ -19,11 +19,8 @@ import functools
 import re
 import socket
 import sys
+from io import BytesIO
 
-try:
-    from io import BytesIO  # python 3
-except ImportError:
-    from cStringIO import StringIO as BytesIO  # python 2
 
 try:
     import urlparse  # py2
@@ -37,7 +34,7 @@ except ImportError:
     ssl = None
 
 try:
-    import certifi
+    import lib.certifi
 except ImportError:
     certifi = None
 
@@ -222,6 +219,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
                     stack_context.wrap(self._on_timeout))
             self.tcp_client.connect(host, port, af=af,
                                     ssl_options=ssl_options,
+                                    max_buffer_size=self.max_buffer_size,
                                     callback=self._on_connect)
 
     def _get_ssl_options(self, scheme):
@@ -277,7 +275,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             stream.close()
             return
         self.stream = stream
-        self.stream.set_close_callback(self._on_close)
+        self.stream.set_close_callback(self.on_connection_close)
         self._remove_timeout()
         if self.final_callback is None:
             return
@@ -316,18 +314,18 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
         if self.request.user_agent:
             self.request.headers["User-Agent"] = self.request.user_agent
         if not self.request.allow_nonstandard_methods:
-            if self.request.method in ("POST", "PATCH", "PUT"):
+            # Some HTTP methods nearly always have bodies while others
-                if (self.request.body is None and
+            # almost never do. Fail in this case unless the user has
-                        self.request.body_producer is None):
+            # opted out of sanity checks with allow_nonstandard_methods.
-                    raise AssertionError(
+            body_expected = self.request.method in ("POST", "PATCH", "PUT")
-                        'Body must not be empty for "%s" request'
+            body_present = (self.request.body is not None or
-                        % self.request.method)
+                            self.request.body_producer is not None)
-            else:
+            if ((body_expected and not body_present) or
-                if (self.request.body is not None or
+                (body_present and not body_expected)):
-                        self.request.body_producer is not None):
+                raise ValueError(
-                    raise AssertionError(
+                    'Body must %sbe None for method %s (unelss '
-                        'Body must be empty for "%s" request'
+                    'allow_nonstandard_methods is true)' %
-                        % self.request.method)
+                    ('not ' if body_expected else '', self.request.method))
         if self.request.expect_100_continue:
             self.request.headers["Expect"] = "100-continue"
         if self.request.body is not None:
@@ -338,7 +336,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
         if (self.request.method == "POST" and
                 "Content-Type" not in self.request.headers):
             self.request.headers["Content-Type"] = "application/x-www-form-urlencoded"
-        if self.request.use_gzip:
+        if self.request.decompress_response:
             self.request.headers["Accept-Encoding"] = "gzip"
         req_path = ((self.parsed.path or '/') +
                     (('?' + self.parsed.query) if self.parsed.query else ''))
@@ -348,7 +346,7 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             HTTP1ConnectionParameters(
                 no_keep_alive=True,
                 max_header_size=self.max_header_size,
-                use_gzip=self.request.use_gzip),
+                decompress=self.request.decompress_response),
             self._sockaddr)
         start_line = httputil.RequestStartLine(self.request.method,
                                                req_path, 'HTTP/1.1')
@@ -418,12 +416,15 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
             # pass it along, unless it's just the stream being closed.
             return isinstance(value, StreamClosedError)
 
-    def _on_close(self):
+    def on_connection_close(self):
         if self.final_callback is not None:
             message = "Connection closed"
             if self.stream.error:
                 raise self.stream.error
-            raise HTTPError(599, message)
+            try:
+                raise HTTPError(599, message)
+            except HTTPError:
+                self._handle_exception(*sys.exc_info())
 
     def headers_received(self, first_line, headers):
         if self.request.expect_100_continue and first_line.code == 100:
@@ -433,20 +434,6 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
         self.code = first_line.code
         self.reason = first_line.reason
 
-        if "Content-Length" in self.headers:
-            if "," in self.headers["Content-Length"]:
-                # Proxies sometimes cause Content-Length headers to get
-                # duplicated.  If all the values are identical then we can
-                # use them but if they differ it's an error.
-                pieces = re.split(r',\s*', self.headers["Content-Length"])
-                if any(i != pieces[0] for i in pieces):
-                    raise ValueError("Multiple unequal Content-Lengths: %r" %
-                                     self.headers["Content-Length"])
-                self.headers["Content-Length"] = pieces[0]
-            content_length = int(self.headers["Content-Length"])
-        else:
-            content_length = None
-
         if self.request.header_callback is not None:
             # Reassemble the start line.
             self.request.header_callback('%s %s %s\r\n' % first_line)
@@ -454,14 +441,6 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
                 self.request.header_callback("%s: %s\r\n" % (k, v))
             self.request.header_callback('\r\n')
 
-        if 100 <= self.code < 200 or self.code == 204:
-            # These response codes never have bodies
-            # http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
-            if ("Transfer-Encoding" in self.headers or
-                    content_length not in (None, 0)):
-                raise ValueError("Response with code %d should not have body" %
-                                 self.code)
-
     def finish(self):
         data = b''.join(self.chunks)
         self._remove_timeout()

tornado/stack_context.py

@@ -41,13 +41,13 @@ Example usage::
             sys.exit(1)
 
     with StackContext(die_on_error):
-        # Any exception thrown here *or in callback and its desendents*
+        # Any exception thrown here *or in callback and its descendants*
         # will cause the process to exit instead of spinning endlessly
         # in the ioloop.
         http_client.fetch(url, callback)
     ioloop.start()
 
-Most applications shouln't have to work with `StackContext` directly.
+Most applications shouldn't have to work with `StackContext` directly.
 Here are a few rules of thumb for when it's necessary:
 
 * If you're writing an asynchronous library that doesn't rely on a

tornado/tcpclient.py

@@ -163,7 +163,7 @@ class TCPClient(object):
             functools.partial(self._create_stream, max_buffer_size))
         af, addr, stream = yield connector.start()
         # TODO: For better performance we could cache the (af, addr)
-        # information here and re-use it on sbusequent connections to
+        # information here and re-use it on subsequent connections to
         # the same host. (http://tools.ietf.org/html/rfc6555#section-4.2)
         if ssl_options is not None:
             stream = yield stream.start_tls(False, ssl_options=ssl_options,

tornado/template.py

@@ -199,7 +199,7 @@ import threading
 
 from tornado import escape
 from tornado.log import app_log
-from tornado.util import bytes_type, ObjectDict, exec_in, unicode_type
+from tornado.util import ObjectDict, exec_in, unicode_type
 
 try:
     from cStringIO import StringIO  # py2
@@ -261,7 +261,7 @@ class Template(object):
             "linkify": escape.linkify,
             "datetime": datetime,
             "_tt_utf8": escape.utf8,  # for internal use
-            "_tt_string_types": (unicode_type, bytes_type),
+            "_tt_string_types": (unicode_type, bytes),
             # __name__ and __loader__ allow the traceback mechanism to find
             # the generated source code.
             "__name__": self.name.replace('.', '_'),

tornado/test/README

@@ -1,4 +1,4 @@
 Test coverage is almost non-existent, but it's a start.  Be sure to
-set PYTHONPATH apprioriately (generally to the root directory of your
+set PYTHONPATH appropriately (generally to the root directory of your
 tornado checkout) when running tests to make sure you're getting the
 version of the tornado package that you expect.

tornado/test/curl_httpclient_test.py

@@ -8,7 +8,8 @@ from tornado.stack_context import ExceptionStackContext
 from tornado.testing import AsyncHTTPTestCase
 from tornado.test import httpclient_test
 from tornado.test.util import unittest
-from tornado.web import Application, RequestHandler
+from tornado.web import Application, RequestHandler, URLSpec
+
 
 try:
     import pycurl

tornado/test/escape_test.py

@@ -4,8 +4,8 @@
 from __future__ import absolute_import, division, print_function, with_statement
 import tornado.escape
 
-from tornado.escape import utf8, xhtml_escape, xhtml_unescape, url_escape, url_unescape, to_unicode, json_decode, json_encode
+from tornado.escape import utf8, xhtml_escape, xhtml_unescape, url_escape, url_unescape, to_unicode, json_decode, json_encode, squeeze, recursive_unicode
-from tornado.util import u, unicode_type, bytes_type
+from tornado.util import u, unicode_type
 from tornado.test.util import unittest
 
 linkify_tests = [
@@ -212,6 +212,22 @@ class EscapeTestCase(unittest.TestCase):
         # convert automatically if they are utf8; on python 3 byte strings
         # are not allowed.
         self.assertEqual(json_decode(json_encode(u("\u00e9"))), u("\u00e9"))
-        if bytes_type is str:
+        if bytes is str:
             self.assertEqual(json_decode(json_encode(utf8(u("\u00e9")))), u("\u00e9"))
             self.assertRaises(UnicodeDecodeError, json_encode, b"\xe9")
+
+    def test_squeeze(self):
+        self.assertEqual(squeeze(u('sequences     of    whitespace   chars'))
+            , u('sequences of whitespace chars'))
+    
+    def test_recursive_unicode(self):
+        tests = {
+            'dict': {b"foo": b"bar"},
+            'list': [b"foo", b"bar"],
+            'tuple': (b"foo", b"bar"),
+            'bytes': b"foo"
+        }
+        self.assertEqual(recursive_unicode(tests['dict']), {u("foo"): u("bar")})
+        self.assertEqual(recursive_unicode(tests['list']), [u("foo"), u("bar")])
+        self.assertEqual(recursive_unicode(tests['tuple']), (u("foo"), u("bar")))
+        self.assertEqual(recursive_unicode(tests['bytes']), u("foo"))

tornado/test/httpclient_test.py

@@ -8,6 +8,8 @@ from contextlib import closing
 import functools
 import sys
 import threading
+import datetime
+from io import BytesIO
 
 from tornado.escape import utf8
 from tornado.httpclient import HTTPRequest, HTTPResponse, _RequestProxy, HTTPError, HTTPClient
@@ -19,13 +21,9 @@ from tornado import netutil
 from tornado.stack_context import ExceptionStackContext, NullContext
 from tornado.testing import AsyncHTTPTestCase, bind_unused_port, gen_test, ExpectLog
 from tornado.test.util import unittest, skipOnTravis
-from tornado.util import u, bytes_type
+from tornado.util import u
 from tornado.web import Application, RequestHandler, url
-
+from tornado.httputil import format_timestamp, HTTPHeaders
-try:
-    from io import BytesIO  # python 3
-except ImportError:
-    from cStringIO import StringIO as BytesIO
 
 
 class HelloWorldHandler(RequestHandler):
@@ -41,6 +39,18 @@ class PostHandler(RequestHandler):
             self.get_argument("arg1"), self.get_argument("arg2")))
 
 
+class PutHandler(RequestHandler):
+    def put(self):
+        self.write("Put body: ")
+        self.write(self.request.body)
+
+
+class RedirectHandler(RequestHandler):
+    def prepare(self):
+        self.redirect(self.get_argument("url"),
+                      status=int(self.get_argument("status", "302")))
+
+
 class ChunkHandler(RequestHandler):
     def get(self):
         self.write("asdf")
@@ -83,6 +93,13 @@ class ContentLength304Handler(RequestHandler):
         pass
 
 
+class PatchHandler(RequestHandler):
+
+    def patch(self):
+        "Return the request payload - so we can check it is being kept"
+        self.write(self.request.body)
+
+
 class AllMethodsHandler(RequestHandler):
     SUPPORTED_METHODS = RequestHandler.SUPPORTED_METHODS + ('OTHER',)
 
@@ -101,6 +118,8 @@ class HTTPClientCommonTestCase(AsyncHTTPTestCase):
         return Application([
             url("/hello", HelloWorldHandler),
             url("/post", PostHandler),
+            url("/put", PutHandler),
+            url("/redirect", RedirectHandler),
             url("/chunk", ChunkHandler),
             url("/auth", AuthHandler),
             url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
@@ -108,8 +127,15 @@ class HTTPClientCommonTestCase(AsyncHTTPTestCase):
             url("/user_agent", UserAgentHandler),
             url("/304_with_content_length", ContentLength304Handler),
             url("/all_methods", AllMethodsHandler),
+            url('/patch', PatchHandler),
         ], gzip=True)
 
+    def test_patch_receives_payload(self):
+        body = b"some patch data"
+        response = self.fetch("/patch", method='PATCH', body=body)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.body, body)
+
     @skipOnTravis
     def test_hello_world(self):
         response = self.fetch("/hello")
@@ -263,7 +289,7 @@ Transfer-Encoding: chunked
 
     def test_types(self):
         response = self.fetch("/hello")
-        self.assertEqual(type(response.body), bytes_type)
+        self.assertEqual(type(response.body), bytes)
         self.assertEqual(type(response.headers["Content-Type"]), str)
         self.assertEqual(type(response.code), int)
         self.assertEqual(type(response.effective_url), str)
@@ -314,10 +340,27 @@ Transfer-Encoding: chunked
         # Construct a new instance of the configured client class
         client = self.http_client.__class__(self.io_loop, force_instance=True,
                                             defaults=defaults)
-        client.fetch(self.get_url('/user_agent'), callback=self.stop)
+        try:
-        response = self.wait()
+            client.fetch(self.get_url('/user_agent'), callback=self.stop)
-        self.assertEqual(response.body, b'TestDefaultUserAgent')
+            response = self.wait()
-        client.close()
+            self.assertEqual(response.body, b'TestDefaultUserAgent')
+        finally:
+            client.close()
+
+    def test_header_types(self):
+        # Header values may be passed as character or utf8 byte strings,
+        # in a plain dictionary or an HTTPHeaders object.
+        # Keys must always be the native str type.
+        # All combinations should have the same results on the wire.
+        for value in [u("MyUserAgent"), b"MyUserAgent"]:
+            for container in [dict, HTTPHeaders]:
+                headers = container()
+                headers['User-Agent'] = value
+                resp = self.fetch('/user_agent', headers=headers)
+                self.assertEqual(
+                    resp.body, b"MyUserAgent",
+                    "response=%r, value=%r, container=%r" %
+                    (resp.body, value, container))
 
     def test_304_with_content_length(self):
         # According to the spec 304 responses SHOULD NOT include
@@ -388,17 +431,39 @@ Transfer-Encoding: chunked
         self.assertEqual(response.body, b'OTHER')
 
     @gen_test
-    def test_body(self):
+    def test_body_sanity_checks(self):
         hello_url = self.get_url('/hello')
-        with self.assertRaises(AssertionError) as context:
+        with self.assertRaises(ValueError) as context:
             yield self.http_client.fetch(hello_url, body='data')
 
-        self.assertTrue('must be empty' in str(context.exception))
+        self.assertTrue('must be None' in str(context.exception))
 
-        with self.assertRaises(AssertionError) as context:
+        with self.assertRaises(ValueError) as context:
             yield self.http_client.fetch(hello_url, method='POST')
 
-        self.assertTrue('must not be empty' in str(context.exception))
+        self.assertTrue('must not be None' in str(context.exception))
+
+    # This test causes odd failures with the combination of
+    # curl_httpclient (at least with the version of libcurl available
+    # on ubuntu 12.04), TwistedIOLoop, and epoll.  For POST (but not PUT),
+    # curl decides the response came back too soon and closes the connection
+    # to start again.  It does this *before* telling the socket callback to
+    # unregister the FD.  Some IOLoop implementations have special kernel
+    # integration to discover this immediately.  Tornado's IOLoops
+    # ignore errors on remove_handler to accommodate this behavior, but
+    # Twisted's reactor does not.  The removeReader call fails and so
+    # do all future removeAll calls (which our tests do at cleanup).
+    #
+    #def test_post_307(self):
+    #    response = self.fetch("/redirect?status=307&url=/post",
+    #                          method="POST", body=b"arg1=foo&arg2=bar")
+    #    self.assertEqual(response.body, b"Post arg1: foo, arg2: bar")
+
+    def test_put_307(self):
+        response = self.fetch("/redirect?status=307&url=/put",
+                              method="PUT", body=b"hello")
+        response.rethrow()
+        self.assertEqual(response.body, b"Put body: hello")
 
 
 class RequestProxyTest(unittest.TestCase):
@@ -515,3 +580,9 @@ class HTTPRequestTestCase(unittest.TestCase):
         request = HTTPRequest('http://example.com')
         request.body = 'foo'
         self.assertEqual(request.body, utf8('foo'))
+
+    def test_if_modified_since(self):
+        http_date = datetime.datetime.utcnow()
+        request = HTTPRequest('http://example.com', if_modified_since=http_date)
+        self.assertEqual(request.headers, 
+            {'If-Modified-Since': format_timestamp(http_date)})

tornado/test/httpserver_test.py

@@ -9,12 +9,12 @@ from tornado.http1connection import HTTP1Connection
 from tornado.httpserver import HTTPServer
 from tornado.httputil import HTTPHeaders, HTTPMessageDelegate, HTTPServerConnectionDelegate, ResponseStartLine
 from tornado.iostream import IOStream
-from tornado.log import gen_log, app_log
+from tornado.log import gen_log
 from tornado.netutil import ssl_options_to_context
 from tornado.simple_httpclient import SimpleAsyncHTTPClient
 from tornado.testing import AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, ExpectLog, gen_test
 from tornado.test.util import unittest, skipOnTravis
-from tornado.util import u, bytes_type
+from tornado.util import u
 from tornado.web import Application, RequestHandler, asynchronous, stream_request_body
 from contextlib import closing
 import datetime
@@ -25,11 +25,7 @@ import socket
 import ssl
 import sys
 import tempfile
-
+from io import BytesIO
-try:
-    from io import BytesIO  # python 3
-except ImportError:
-    from cStringIO import StringIO as BytesIO  # python 2
 
 
 def read_stream_body(stream, callback):
@@ -297,10 +293,10 @@ class TypeCheckHandler(RequestHandler):
         # secure cookies
 
         self.check_type('arg_key', list(self.request.arguments.keys())[0], str)
-        self.check_type('arg_value', list(self.request.arguments.values())[0][0], bytes_type)
+        self.check_type('arg_value', list(self.request.arguments.values())[0][0], bytes)
 
     def post(self):
-        self.check_type('body', self.request.body, bytes_type)
+        self.check_type('body', self.request.body, bytes)
         self.write(self.errors)
 
     def get(self):
@@ -358,7 +354,7 @@ class HTTPServerTest(AsyncHTTPTestCase):
         # if the data is not utf8.  On python 2 parse_qs will work,
         # but then the recursive_unicode call in EchoHandler will
         # fail.
-        if str is bytes_type:
+        if str is bytes:
             return
         with ExpectLog(gen_log, 'Invalid x-www-form-urlencoded body'):
             response = self.fetch(
@@ -586,6 +582,8 @@ class KeepAliveTest(AsyncHTTPTestCase):
         class HelloHandler(RequestHandler):
             def get(self):
                 self.finish('Hello world')
+            def post(self):
+                self.finish('Hello world')
 
         class LargeHandler(RequestHandler):
             def get(self):
@@ -687,6 +685,17 @@ class KeepAliveTest(AsyncHTTPTestCase):
         self.assertEqual(self.headers['Connection'], 'Keep-Alive')
         self.close()
 
+    def test_http10_keepalive_extra_crlf(self):
+        self.http_version = b'HTTP/1.0'
+        self.connect()
+        self.stream.write(b'GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n\r\n')
+        self.read_response()
+        self.assertEqual(self.headers['Connection'], 'Keep-Alive')
+        self.stream.write(b'GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n')
+        self.read_response()
+        self.assertEqual(self.headers['Connection'], 'Keep-Alive')
+        self.close()
+
     def test_pipelined_requests(self):
         self.connect()
         self.stream.write(b'GET / HTTP/1.1\r\n\r\nGET / HTTP/1.1\r\n\r\n')
@@ -715,6 +724,19 @@ class KeepAliveTest(AsyncHTTPTestCase):
         self.read_headers()
         self.close()
 
+    def test_keepalive_chunked(self):
+        self.http_version = b'HTTP/1.0'
+        self.connect()
+        self.stream.write(b'POST / HTTP/1.0\r\nConnection: keep-alive\r\n'
+                          b'Transfer-Encoding: chunked\r\n'
+                          b'\r\n0\r\n')
+        self.read_response()
+        self.assertEqual(self.headers['Connection'], 'Keep-Alive')
+        self.stream.write(b'GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n')
+        self.read_response()
+        self.assertEqual(self.headers['Connection'], 'Keep-Alive')
+        self.close()
+
 
 class GzipBaseTest(object):
     def get_app(self):
@@ -736,7 +758,7 @@ class GzipBaseTest(object):
 
 class GzipTest(GzipBaseTest, AsyncHTTPTestCase):
     def get_httpserver_options(self):
-        return dict(gzip=True)
+        return dict(decompress_request=True)
 
     def test_gzip(self):
         response = self.post_gzip('foo=bar')
@@ -764,7 +786,7 @@ class StreamingChunkSizeTest(AsyncHTTPTestCase):
         return SimpleAsyncHTTPClient(io_loop=self.io_loop)
 
     def get_httpserver_options(self):
-        return dict(chunk_size=self.CHUNK_SIZE, gzip=True)
+        return dict(chunk_size=self.CHUNK_SIZE, decompress_request=True)
 
     class MessageDelegate(HTTPMessageDelegate):
         def __init__(self, connection):

tornado/test/httputil_test.py

@@ -2,7 +2,7 @@
 
 
 from __future__ import absolute_import, division, print_function, with_statement
-from tornado.httputil import url_concat, parse_multipart_form_data, HTTPHeaders, format_timestamp
+from tornado.httputil import url_concat, parse_multipart_form_data, HTTPHeaders, format_timestamp, HTTPServerRequest, parse_request_start_line
 from tornado.escape import utf8
 from tornado.log import gen_log
 from tornado.testing import ExpectLog
@@ -253,3 +253,26 @@ class FormatTimestampTest(unittest.TestCase):
 
     def test_datetime(self):
         self.check(datetime.datetime.utcfromtimestamp(self.TIMESTAMP))
+
+
+# HTTPServerRequest is mainly tested incidentally to the server itself,
+# but this tests the parts of the class that can be tested in isolation.
+class HTTPServerRequestTest(unittest.TestCase):
+    def test_default_constructor(self):
+        # All parameters are formally optional, but uri is required
+        # (and has been for some time).  This test ensures that no
+        # more required parameters slip in.
+        HTTPServerRequest(uri='/')
+
+
+class ParseRequestStartLineTest(unittest.TestCase):
+    METHOD = "GET"
+    PATH = "/foo"
+    VERSION = "HTTP/1.1"
+
+    def test_parse_request_start_line(self):
+        start_line = " ".join([self.METHOD, self.PATH, self.VERSION])
+        parsed_start_line = parse_request_start_line(start_line)
+        self.assertEqual(parsed_start_line.method, self.METHOD)
+        self.assertEqual(parsed_start_line.path, self.PATH)
+        self.assertEqual(parsed_start_line.version, self.VERSION)

tornado/test/ioloop_test.py

@@ -173,6 +173,25 @@ class TestIOLoop(AsyncTestCase):
         self.io_loop.add_callback(lambda: self.io_loop.add_callback(self.stop))
         self.wait()
 
+    def test_remove_timeout_from_timeout(self):
+        calls = [False, False]
+
+        # Schedule several callbacks and wait for them all to come due at once.
+        # t2 should be cancelled by t1, even though it is already scheduled to
+        # be run before the ioloop even looks at it.
+        now = self.io_loop.time()
+        def t1():
+            calls[0] = True
+            self.io_loop.remove_timeout(t2_handle)
+        self.io_loop.add_timeout(now + 0.01, t1)
+        def t2():
+            calls[1] = True
+        t2_handle = self.io_loop.add_timeout(now + 0.02, t2)
+        self.io_loop.add_timeout(now + 0.03, self.stop)
+        time.sleep(0.03)
+        self.wait()
+        self.assertEqual(calls, [True, False])
+
     def test_timeout_with_arguments(self):
         # This tests that all the timeout methods pass through *args correctly.
         results = []
@@ -185,6 +204,23 @@ class TestIOLoop(AsyncTestCase):
         self.wait()
         self.assertEqual(results, [1, 2, 3, 4])
 
+    def test_add_timeout_return(self):
+        # All the timeout methods return non-None handles that can be
+        # passed to remove_timeout.
+        handle = self.io_loop.add_timeout(self.io_loop.time(), lambda: None)
+        self.assertFalse(handle is None)
+        self.io_loop.remove_timeout(handle)
+
+    def test_call_at_return(self):
+        handle = self.io_loop.call_at(self.io_loop.time(), lambda: None)
+        self.assertFalse(handle is None)
+        self.io_loop.remove_timeout(handle)
+
+    def test_call_later_return(self):
+        handle = self.io_loop.call_later(0, lambda: None)
+        self.assertFalse(handle is None)
+        self.io_loop.remove_timeout(handle)
+
     def test_close_file_object(self):
         """When a file object is used instead of a numeric file descriptor,
         the object should be closed (by IOLoop.close(all_fds=True),
@@ -298,6 +334,33 @@ class TestIOLoop(AsyncTestCase):
         with ExpectLog(app_log, "Exception in callback"):
             self.wait()
 
+    @skipIfNonUnix
+    def test_remove_handler_from_handler(self):
+        # Create two sockets with simultaneous read events.
+        client, server = socket.socketpair()
+        try:
+            client.send(b'abc')
+            server.send(b'abc')
+
+            # After reading from one fd, remove the other from the IOLoop.
+            chunks = []
+            def handle_read(fd, events):
+                chunks.append(fd.recv(1024))
+                if fd is client:
+                    self.io_loop.remove_handler(server)
+                else:
+                    self.io_loop.remove_handler(client)
+            self.io_loop.add_handler(client, handle_read, self.io_loop.READ)
+            self.io_loop.add_handler(server, handle_read, self.io_loop.READ)
+            self.io_loop.call_later(0.01, self.stop)
+            self.wait()
+
+            # Only one fd was read; the other was cleanly removed.
+            self.assertEqual(chunks, [b'abc'])
+        finally:
+            client.close()
+            server.close()
+
 
 # Deliberately not a subclass of AsyncTestCase so the IOLoop isn't
 # automatically set as current.

tornado/test/iostream_test.py

@@ -10,7 +10,7 @@ from tornado.stack_context import NullContext
 from tornado.testing import AsyncHTTPTestCase, AsyncHTTPSTestCase, AsyncTestCase, bind_unused_port, ExpectLog, gen_test
 from tornado.test.util import unittest, skipIfNonUnix
 from tornado.web import RequestHandler, Application
-import certifi
+import lib.certifi
 import errno
 import logging
 import os
@@ -511,7 +511,7 @@ class TestIOStreamMixin(object):
         server, client = self.make_iostream_pair()
         server.set_close_callback(self.stop)
         try:
-            # Start a read that will be fullfilled asynchronously.
+            # Start a read that will be fulfilled asynchronously.
             server.read_bytes(1, lambda data: None)
             client.write(b'a')
             # Stub out read_from_fd to make it fail.

tornado/test/locale_test.py

@@ -57,3 +57,43 @@ class EnglishTest(unittest.TestCase):
         date = datetime.datetime(2013, 4, 28, 18, 35)
         self.assertEqual(locale.format_date(date, full_format=True),
                          'April 28, 2013 at 6:35 pm')
+
+        self.assertEqual(locale.format_date(datetime.datetime.utcnow() - datetime.timedelta(seconds=2), full_format=False), 
+                         '2 seconds ago')
+        self.assertEqual(locale.format_date(datetime.datetime.utcnow() - datetime.timedelta(minutes=2), full_format=False),
+                         '2 minutes ago')
+        self.assertEqual(locale.format_date(datetime.datetime.utcnow() - datetime.timedelta(hours=2), full_format=False),
+                         '2 hours ago')
+
+        now = datetime.datetime.utcnow()
+        self.assertEqual(locale.format_date(now - datetime.timedelta(days=1), full_format=False, shorter=True),
+                         'yesterday')
+
+        date = now - datetime.timedelta(days=2)
+        self.assertEqual(locale.format_date(date, full_format=False, shorter=True),
+                         locale._weekdays[date.weekday()])
+
+        date = now - datetime.timedelta(days=300)
+        self.assertEqual(locale.format_date(date, full_format=False, shorter=True),
+                         '%s %d' % (locale._months[date.month - 1], date.day))
+
+        date = now - datetime.timedelta(days=500)
+        self.assertEqual(locale.format_date(date, full_format=False, shorter=True),
+                         '%s %d, %d' % (locale._months[date.month - 1], date.day, date.year))
+
+    def test_friendly_number(self):
+        locale = tornado.locale.get('en_US')
+        self.assertEqual(locale.friendly_number(1000000), '1,000,000')
+
+    def test_list(self):
+        locale = tornado.locale.get('en_US')
+        self.assertEqual(locale.list([]), '')
+        self.assertEqual(locale.list(['A']), 'A')
+        self.assertEqual(locale.list(['A', 'B']), 'A and B')
+        self.assertEqual(locale.list(['A', 'B', 'C']), 'A, B and C')
+
+    def test_format_day(self):
+        locale = tornado.locale.get('en_US')
+        date = datetime.datetime(2013, 4, 28, 18, 35)
+        self.assertEqual(locale.format_day(date=date, dow=True), 'Sunday, April 28')
+        self.assertEqual(locale.format_day(date=date, dow=False), 'April 28')

tornado/test/log_test.py

@@ -29,7 +29,7 @@ from tornado.escape import utf8
 from tornado.log import LogFormatter, define_logging_options, enable_pretty_logging
 from tornado.options import OptionParser
 from tornado.test.util import unittest
-from tornado.util import u, bytes_type, basestring_type
+from tornado.util import u, basestring_type
 
 
 @contextlib.contextmanager
@@ -95,8 +95,9 @@ class LogFormatterTest(unittest.TestCase):
             self.assertEqual(self.get_output(), utf8(repr(b"\xe9")))
 
     def test_utf8_logging(self):
-        self.logger.error(u("\u00e9").encode("utf8"))
+        with ignore_bytes_warning():
-        if issubclass(bytes_type, basestring_type):
+            self.logger.error(u("\u00e9").encode("utf8"))
+        if issubclass(bytes, basestring_type):
             # on python 2, utf8 byte strings (and by extension ascii byte
             # strings) are passed through as-is.
             self.assertEqual(self.get_output(), utf8(u("\u00e9")))

tornado/test/netutil_test.py

@@ -34,15 +34,6 @@ else:
 
 
 class _ResolverTestMixin(object):
-    def skipOnCares(self):
-        # Some DNS-hijacking ISPs (e.g. Time Warner) return non-empty results
-        # with an NXDOMAIN status code.  Most resolvers treat this as an error;
-        # C-ares returns the results, making the "bad_host" tests unreliable.
-        # C-ares will try to resolve even malformed names, such as the
-        # name with spaces used in this test.
-        if self.resolver.__class__.__name__ == 'CaresResolver':
-            self.skipTest("CaresResolver doesn't recognize fake NXDOMAIN")
-
     def test_localhost(self):
         self.resolver.resolve('localhost', 80, callback=self.stop)
         result = self.wait()
@@ -55,8 +46,11 @@ class _ResolverTestMixin(object):
         self.assertIn((socket.AF_INET, ('127.0.0.1', 80)),
                       addrinfo)
 
+
+# It is impossible to quickly and consistently generate an error in name
+# resolution, so test this case separately, using mocks as needed.
+class _ResolverErrorTestMixin(object):
     def test_bad_host(self):
-        self.skipOnCares()
         def handler(exc_typ, exc_val, exc_tb):
             self.stop(exc_val)
             return True  # Halt propagation.
@@ -69,11 +63,13 @@ class _ResolverTestMixin(object):
 
     @gen_test
     def test_future_interface_bad_host(self):
-        self.skipOnCares()
         with self.assertRaises(Exception):
             yield self.resolver.resolve('an invalid domain', 80,
                                         socket.AF_UNSPEC)
 
+def _failing_getaddrinfo(*args):
+    """Dummy implementation of getaddrinfo for use in mocks"""
+    raise socket.gaierror("mock: lookup failed")
 
 @skipIfNoNetwork
 class BlockingResolverTest(AsyncTestCase, _ResolverTestMixin):
@@ -82,6 +78,21 @@ class BlockingResolverTest(AsyncTestCase, _ResolverTestMixin):
         self.resolver = BlockingResolver(io_loop=self.io_loop)
 
 
+# getaddrinfo-based tests need mocking to reliably generate errors;
+# some configurations are slow to produce errors and take longer than
+# our default timeout.
+class BlockingResolverErrorTest(AsyncTestCase, _ResolverErrorTestMixin):
+    def setUp(self):
+        super(BlockingResolverErrorTest, self).setUp()
+        self.resolver = BlockingResolver(io_loop=self.io_loop)
+        self.real_getaddrinfo = socket.getaddrinfo
+        socket.getaddrinfo = _failing_getaddrinfo
+
+    def tearDown(self):
+        socket.getaddrinfo = self.real_getaddrinfo
+        super(BlockingResolverErrorTest, self).tearDown()
+
+
 @skipIfNoNetwork
 @unittest.skipIf(futures is None, "futures module not present")
 class ThreadedResolverTest(AsyncTestCase, _ResolverTestMixin):
@@ -94,6 +105,18 @@ class ThreadedResolverTest(AsyncTestCase, _ResolverTestMixin):
         super(ThreadedResolverTest, self).tearDown()
 
 
+class ThreadedResolverErrorTest(AsyncTestCase, _ResolverErrorTestMixin):
+    def setUp(self):
+        super(ThreadedResolverErrorTest, self).setUp()
+        self.resolver = BlockingResolver(io_loop=self.io_loop)
+        self.real_getaddrinfo = socket.getaddrinfo
+        socket.getaddrinfo = _failing_getaddrinfo
+
+    def tearDown(self):
+        socket.getaddrinfo = self.real_getaddrinfo
+        super(ThreadedResolverErrorTest, self).tearDown()
+
+
 @skipIfNoNetwork
 @unittest.skipIf(futures is None, "futures module not present")
 @unittest.skipIf(sys.platform == 'win32', "preexec_fn not available on win32")
@@ -121,6 +144,12 @@ class ThreadedResolverImportTest(unittest.TestCase):
         self.fail("import timed out")
 
 
+# We do not test errors with CaresResolver:
+# Some DNS-hijacking ISPs (e.g. Time Warner) return non-empty results
+# with an NXDOMAIN status code.  Most resolvers treat this as an error;
+# C-ares returns the results, making the "bad_host" tests unreliable.
+# C-ares will try to resolve even malformed names, such as the
+# name with spaces used in this test.
 @skipIfNoNetwork
 @unittest.skipIf(pycares is None, "pycares module not present")
 class CaresResolverTest(AsyncTestCase, _ResolverTestMixin):
@@ -129,10 +158,13 @@ class CaresResolverTest(AsyncTestCase, _ResolverTestMixin):
         self.resolver = CaresResolver(io_loop=self.io_loop)
 
 
+# TwistedResolver produces consistent errors in our test cases so we
+# can test the regular and error cases in the same class.
 @skipIfNoNetwork
 @unittest.skipIf(twisted is None, "twisted module not present")
 @unittest.skipIf(getattr(twisted, '__version__', '0.0') < "12.1", "old version of twisted")
-class TwistedResolverTest(AsyncTestCase, _ResolverTestMixin):
+class TwistedResolverTest(AsyncTestCase, _ResolverTestMixin,
+                          _ResolverErrorTestMixin):
     def setUp(self):
         super(TwistedResolverTest, self).setUp()
         self.resolver = TwistedResolver(io_loop=self.io_loop)

tornado/test/options_test.cfg

@@ -1,2 +1,3 @@
 port=443
 port=443
+username='李康'

tornado/test/options_test.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function, with_statement
 
 import datetime
@@ -32,9 +33,11 @@ class OptionsTest(unittest.TestCase):
     def test_parse_config_file(self):
         options = OptionParser()
         options.define("port", default=80)
+        options.define("username", default='foo')
         options.parse_config_file(os.path.join(os.path.dirname(__file__),
                                                "options_test.cfg"))
         self.assertEquals(options.port, 443)
+        self.assertEqual(options.username, "李康")
 
     def test_parse_callbacks(self):
         options = OptionParser()

tornado/test/simple_httpclient_test.py

@@ -14,7 +14,7 @@ from tornado import gen
 from tornado.httpclient import AsyncHTTPClient
 from tornado.httputil import HTTPHeaders
 from tornado.ioloop import IOLoop
-from tornado.log import gen_log, app_log
+from tornado.log import gen_log
 from tornado.netutil import Resolver, bind_sockets
 from tornado.simple_httpclient import SimpleAsyncHTTPClient, _default_ca_certs
 from tornado.test.httpclient_test import ChunkHandler, CountdownHandler, HelloWorldHandler
@@ -294,10 +294,13 @@ class SimpleHTTPClientTestMixin(object):
         self.assertEqual(response.code, 204)
         # 204 status doesn't need a content-length, but tornado will
         # add a zero content-length anyway.
+        #
+        # A test without a content-length header is included below
+        # in HTTP204NoContentTestCase.
         self.assertEqual(response.headers["Content-length"], "0")
 
         # 204 status with non-zero content length is malformed
-        with ExpectLog(app_log, "Uncaught exception"):
+        with ExpectLog(gen_log, "Malformed HTTP message"):
             response = self.fetch("/no_content?error=1")
         self.assertEqual(response.code, 599)
 
@@ -476,6 +479,27 @@ class HTTP100ContinueTestCase(AsyncHTTPTestCase):
         self.assertEqual(res.body, b'A')
 
 
+class HTTP204NoContentTestCase(AsyncHTTPTestCase):
+    def respond_204(self, request):
+        # A 204 response never has a body, even if doesn't have a content-length
+        # (which would otherwise mean read-until-close).  Tornado always
+        # sends a content-length, so we simulate here a server that sends
+        # no content length and does not close the connection.
+        #
+        # Tests of a 204 response with a Content-Length header are included
+        # in SimpleHTTPClientTestMixin.
+        request.connection.stream.write(
+            b"HTTP/1.1 204 No content\r\n\r\n")
+
+    def get_app(self):
+        return self.respond_204
+
+    def test_204_no_content(self):
+        resp = self.fetch('/')
+        self.assertEqual(resp.code, 204)
+        self.assertEqual(resp.body, b'')
+
+
 class HostnameMappingTestCase(AsyncHTTPTestCase):
     def setUp(self):
         super(HostnameMappingTestCase, self).setUp()

tornado/test/tcpclient_test.py

@@ -72,7 +72,9 @@ class TCPClientTest(AsyncTestCase):
         super(TCPClientTest, self).tearDown()
 
     def skipIfLocalhostV4(self):
-        Resolver().resolve('localhost', 0, callback=self.stop)
+        # The port used here doesn't matter, but some systems require it
+        # to be non-zero if we do not also pass AI_PASSIVE.
+        Resolver().resolve('localhost', 80, callback=self.stop)
         addrinfo = self.wait()
         families = set(addr[0] for addr in addrinfo)
         if socket.AF_INET6 not in families:

tornado/test/template_test.py

@@ -7,7 +7,7 @@ import traceback
 from tornado.escape import utf8, native_str, to_unicode
 from tornado.template import Template, DictLoader, ParseError, Loader
 from tornado.test.util import unittest
-from tornado.util import u, bytes_type, ObjectDict, unicode_type
+from tornado.util import u, ObjectDict, unicode_type
 
 
 class TemplateTest(unittest.TestCase):
@@ -374,7 +374,7 @@ raw: {% raw name %}""",
                              "{% autoescape py_escape %}s = {{ name }}\n"})
 
         def py_escape(s):
-            self.assertEqual(type(s), bytes_type)
+            self.assertEqual(type(s), bytes)
             return repr(native_str(s))
 
         def render(template, name):

tornado/test/testing_test.py

@@ -3,7 +3,8 @@
 from __future__ import absolute_import, division, print_function, with_statement
 
 from tornado import gen, ioloop
-from tornado.testing import AsyncTestCase, gen_test
+from tornado.log import app_log
+from tornado.testing import AsyncTestCase, gen_test, ExpectLog
 from tornado.test.util import unittest
 
 import contextlib
@@ -13,7 +14,7 @@ import traceback
 
 @contextlib.contextmanager
 def set_environ(name, value):
-    old_value = os.environ.get('name')
+    old_value = os.environ.get(name)
     os.environ[name] = value
 
     try:
@@ -62,6 +63,17 @@ class AsyncTestCaseTest(AsyncTestCase):
         self.io_loop.add_timeout(self.io_loop.time() + 0.03, self.stop)
         self.wait(timeout=0.15)
 
+    def test_multiple_errors(self):
+        def fail(message):
+            raise Exception(message)
+        self.io_loop.add_callback(lambda: fail("error one"))
+        self.io_loop.add_callback(lambda: fail("error two"))
+        # The first error gets raised; the second gets logged.
+        with ExpectLog(app_log, "multiple unhandled exceptions"):
+            with self.assertRaises(Exception) as cm:
+                self.wait()
+        self.assertEqual(str(cm.exception), "error one")
+
 
 class AsyncTestCaseWrapperTest(unittest.TestCase):
     def test_undecorated_generator(self):

tornado/test/util_test.py

@@ -1,9 +1,10 @@
 # coding: utf-8
 from __future__ import absolute_import, division, print_function, with_statement
 import sys
+import datetime
 
 from tornado.escape import utf8
-from tornado.util import raise_exc_info, Configurable, u, exec_in, ArgReplacer
+from tornado.util import raise_exc_info, Configurable, u, exec_in, ArgReplacer, timedelta_to_seconds
 from tornado.test.util import unittest
 
 try:
@@ -170,3 +171,9 @@ class ArgReplacerTest(unittest.TestCase):
         self.assertEqual(self.replacer.get_old_value(args, kwargs), 'old')
         self.assertEqual(self.replacer.replace('new', args, kwargs),
                          ('old', (1,), dict(y=2, callback='new', z=3)))
+
+
+class TimedeltaToSecondsTest(unittest.TestCase):
+    def test_timedelta_to_seconds(self):
+        time_delta = datetime.timedelta(hours=1)
+        self.assertEqual(timedelta_to_seconds(time_delta), 3600.0)

tornado/test/web_test.py

@@ -4,13 +4,14 @@ from tornado import gen
 from tornado.escape import json_decode, utf8, to_unicode, recursive_unicode, native_str, to_basestring
 from tornado.httputil import format_timestamp
 from tornado.iostream import IOStream
+from tornado import locale
 from tornado.log import app_log, gen_log
 from tornado.simple_httpclient import SimpleAsyncHTTPClient
 from tornado.template import DictLoader
 from tornado.testing import AsyncHTTPTestCase, ExpectLog, gen_test
 from tornado.test.util import unittest
-from tornado.util import u, bytes_type, ObjectDict, unicode_type
+from tornado.util import u, ObjectDict, unicode_type, timedelta_to_seconds
-from tornado.web import RequestHandler, authenticated, Application, asynchronous, url, HTTPError, StaticFileHandler, _create_signature_v1, create_signed_value, decode_signed_value, ErrorHandler, UIModule, MissingArgumentError, stream_request_body, Finish
+from tornado.web import RequestHandler, authenticated, Application, asynchronous, url, HTTPError, StaticFileHandler, _create_signature_v1, create_signed_value, decode_signed_value, ErrorHandler, UIModule, MissingArgumentError, stream_request_body, Finish, removeslash, addslash, RedirectHandler as WebRedirectHandler
 
 import binascii
 import contextlib
@@ -21,7 +22,6 @@ import logging
 import os
 import re
 import socket
-import sys
 
 try:
     import urllib.parse as urllib_parse  # py3
@@ -163,11 +163,21 @@ class CookieTest(WebTestCase):
                 # Attributes from the first call are not carried over.
                 self.set_cookie("a", "e")
 
+        class SetCookieMaxAgeHandler(RequestHandler):
+            def get(self):
+                self.set_cookie("foo", "bar", max_age=10)
+
+        class SetCookieExpiresDaysHandler(RequestHandler):
+            def get(self):
+                self.set_cookie("foo", "bar", expires_days=10)
+
         return [("/set", SetCookieHandler),
                 ("/get", GetCookieHandler),
                 ("/set_domain", SetCookieDomainHandler),
                 ("/special_char", SetCookieSpecialCharHandler),
                 ("/set_overwrite", SetCookieOverwriteHandler),
+                ("/set_max_age", SetCookieMaxAgeHandler),
+                ("/set_expires_days", SetCookieExpiresDaysHandler),
                 ]
 
     def test_set_cookie(self):
@@ -222,6 +232,23 @@ class CookieTest(WebTestCase):
         self.assertEqual(sorted(headers),
                          ["a=e; Path=/", "c=d; Domain=example.com; Path=/"])
 
+    def test_set_cookie_max_age(self):
+        response = self.fetch("/set_max_age")
+        headers = response.headers.get_list("Set-Cookie")
+        self.assertEqual(sorted(headers),
+                         ["foo=bar; Max-Age=10; Path=/"])
+    
+    def test_set_cookie_expires_days(self):
+        response = self.fetch("/set_expires_days")
+        header = response.headers.get("Set-Cookie")
+        match = re.match("foo=bar; expires=(?P<expires>.+); Path=/", header)
+        self.assertIsNotNone(match)
+
+        expires = datetime.datetime.utcnow() + datetime.timedelta(days=10)
+        header_expires = datetime.datetime(
+            *email.utils.parsedate(match.groupdict()["expires"])[:6])
+        self.assertTrue(abs(timedelta_to_seconds(expires - header_expires)) < 10)
+    
 
 class AuthRedirectRequestHandler(RequestHandler):
     def initialize(self, login_url):
@@ -302,7 +329,7 @@ class EchoHandler(RequestHandler):
             if type(key) != str:
                 raise Exception("incorrect type for key: %r" % type(key))
             for value in self.request.arguments[key]:
-                if type(value) != bytes_type:
+                if type(value) != bytes:
                     raise Exception("incorrect type for value: %r" %
                                     type(value))
             for value in self.get_arguments(key):
@@ -370,10 +397,10 @@ class TypeCheckHandler(RequestHandler):
         if list(self.cookies.keys()) != ['asdf']:
             raise Exception("unexpected values for cookie keys: %r" %
                             self.cookies.keys())
-        self.check_type('get_secure_cookie', self.get_secure_cookie('asdf'), bytes_type)
+        self.check_type('get_secure_cookie', self.get_secure_cookie('asdf'), bytes)
         self.check_type('get_cookie', self.get_cookie('asdf'), str)
 
-        self.check_type('xsrf_token', self.xsrf_token, bytes_type)
+        self.check_type('xsrf_token', self.xsrf_token, bytes)
         self.check_type('xsrf_form_html', self.xsrf_form_html(), str)
 
         self.check_type('reverse_url', self.reverse_url('typecheck', 'foo'), str)
@@ -399,7 +426,7 @@ class TypeCheckHandler(RequestHandler):
 
 class DecodeArgHandler(RequestHandler):
     def decode_argument(self, value, name=None):
-        if type(value) != bytes_type:
+        if type(value) != bytes:
             raise Exception("unexpected type for value: %r" % type(value))
         # use self.request.arguments directly to avoid recursion
         if 'encoding' in self.request.arguments:
@@ -409,7 +436,7 @@ class DecodeArgHandler(RequestHandler):
 
     def get(self, arg):
         def describe(s):
-            if type(s) == bytes_type:
+            if type(s) == bytes:
                 return ["bytes", native_str(binascii.b2a_hex(s))]
             elif type(s) == unicode_type:
                 return ["unicode", s]
@@ -550,6 +577,8 @@ class WSGISafeWebTest(WebTestCase):
             url("/optional_path/(.+)?", OptionalPathHandler),
             url("/multi_header", MultiHeaderHandler),
             url("/redirect", RedirectHandler),
+            url("/web_redirect_permanent", WebRedirectHandler, {"url": "/web_redirect_newpath"}),
+            url("/web_redirect", WebRedirectHandler, {"url": "/web_redirect_newpath", "permanent": False}),
             url("/header_injection", HeaderInjectionHandler),
             url("/get_argument", GetArgumentHandler),
             url("/get_arguments", GetArgumentsHandler),
@@ -675,6 +704,14 @@ js_embed()
         response = self.fetch("/redirect?status=307", follow_redirects=False)
         self.assertEqual(response.code, 307)
 
+    def test_web_redirect(self):
+        response = self.fetch("/web_redirect_permanent", follow_redirects=False)
+        self.assertEqual(response.code, 301)
+        self.assertEqual(response.headers['Location'], '/web_redirect_newpath')
+        response = self.fetch("/web_redirect", follow_redirects=False)
+        self.assertEqual(response.code, 302)
+        self.assertEqual(response.headers['Location'], '/web_redirect_newpath')
+
     def test_header_injection(self):
         response = self.fetch("/header_injection")
         self.assertEqual(response.body, b"ok")
@@ -1348,7 +1385,9 @@ class GzipTestCase(SimpleHandlerTestCase):
             self.write('hello world')
 
     def get_app_kwargs(self):
-        return dict(gzip=True)
+        return dict(
+            gzip=True,
+            static_path=os.path.join(os.path.dirname(__file__), 'static'))
 
     def test_gzip(self):
         response = self.fetch('/')
@@ -1361,6 +1400,17 @@ class GzipTestCase(SimpleHandlerTestCase):
             'gzip')
         self.assertEqual(response.headers['Vary'], 'Accept-Encoding')
 
+    def test_gzip_static(self):
+        # The streaming responses in StaticFileHandler have subtle
+        # interactions with the gzip output so test this case separately.
+        response = self.fetch('/robots.txt')
+        self.assertEqual(
+            response.headers.get(
+                'Content-Encoding',
+                response.headers.get('X-Consumed-Content-Encoding')),
+            'gzip')
+        self.assertEqual(response.headers['Vary'], 'Accept-Encoding')
+
     def test_gzip_not_requested(self):
         response = self.fetch('/', use_gzip=False)
         self.assertNotIn('Content-Encoding', response.headers)
@@ -1554,19 +1604,26 @@ class MultipleExceptionTest(SimpleHandlerTestCase):
 
 
 @wsgi_safe
-class SetCurrentUserTest(SimpleHandlerTestCase):
+class SetLazyPropertiesTest(SimpleHandlerTestCase):
     class Handler(RequestHandler):
         def prepare(self):
             self.current_user = 'Ben'
+            self.locale = locale.get('en_US')
+
+        def get_user_locale(self):
+            raise NotImplementedError()
+
+        def get_current_user(self):
+            raise NotImplementedError()
 
         def get(self):
-            self.write('Hello %s' % self.current_user)
+            self.write('Hello %s (%s)' % (self.current_user, self.locale.code))
 
-    def test_set_current_user(self):
+    def test_set_properties(self):
         # Ensure that current_user can be assigned to normally for apps
         # that want to forgo the lazy get_current_user property
         response = self.fetch('/')
-        self.assertEqual(response.body, b'Hello Ben')
+        self.assertEqual(response.body, b'Hello Ben (en_US)')
 
 
 @wsgi_safe
@@ -2193,6 +2250,20 @@ class XSRFTest(SimpleHandlerTestCase):
                 headers=self.cookie_headers())
         self.assertEqual(response.code, 403)
 
+    def test_xsrf_success_short_token(self):
+        response = self.fetch(
+            "/", method="POST",
+            body=urllib_parse.urlencode(dict(_xsrf='deadbeef')),
+            headers=self.cookie_headers(token='deadbeef'))
+        self.assertEqual(response.code, 200)
+
+    def test_xsrf_success_non_hex_token(self):
+        response = self.fetch(
+            "/", method="POST",
+            body=urllib_parse.urlencode(dict(_xsrf='xoxo')),
+            headers=self.cookie_headers(token='xoxo'))
+        self.assertEqual(response.code, 200)
+
     def test_xsrf_success_post_body(self):
         response = self.fetch(
             "/", method="POST",
@@ -2299,3 +2370,38 @@ class FinishExceptionTest(SimpleHandlerTestCase):
         self.assertEqual('Basic realm="something"',
                          response.headers.get('WWW-Authenticate'))
         self.assertEqual(b'authentication required', response.body)
+
+
+class DecoratorTest(WebTestCase):
+    def get_handlers(self):
+        class RemoveSlashHandler(RequestHandler):
+            @removeslash
+            def get(self):
+                pass
+
+        class AddSlashHandler(RequestHandler):
+            @addslash
+            def get(self):
+                pass
+
+        return [("/removeslash/", RemoveSlashHandler),
+                ("/addslash", AddSlashHandler),
+                ]
+
+    def test_removeslash(self):
+        response = self.fetch("/removeslash/", follow_redirects=False)
+        self.assertEqual(response.code, 301)
+        self.assertEqual(response.headers['Location'], "/removeslash")
+
+        response = self.fetch("/removeslash/?foo=bar", follow_redirects=False)
+        self.assertEqual(response.code, 301)
+        self.assertEqual(response.headers['Location'], "/removeslash?foo=bar")
+
+    def test_addslash(self):
+        response = self.fetch("/addslash", follow_redirects=False)
+        self.assertEqual(response.code, 301)
+        self.assertEqual(response.headers['Location'], "/addslash/")
+
+        response = self.fetch("/addslash?foo=bar", follow_redirects=False)
+        self.assertEqual(response.code, 301)
+        self.assertEqual(response.headers['Location'], "/addslash/?foo=bar")

tornado/test/websocket_test.py

@@ -3,11 +3,13 @@ from __future__ import absolute_import, division, print_function, with_statement
 import traceback
 
 from tornado.concurrent import Future
+from tornado import gen
 from tornado.httpclient import HTTPError, HTTPRequest
-from tornado.log import gen_log
+from tornado.log import gen_log, app_log
 from tornado.testing import AsyncHTTPTestCase, gen_test, bind_unused_port, ExpectLog
 from tornado.test.util import unittest
 from tornado.web import Application, RequestHandler
+from tornado.util import u
 
 try:
     import tornado.websocket
@@ -33,8 +35,12 @@ class TestWebSocketHandler(WebSocketHandler):
 
     This allows for deterministic cleanup of the associated socket.
     """
-    def initialize(self, close_future):
+    def initialize(self, close_future, compression_options=None):
         self.close_future = close_future
+        self.compression_options = compression_options
+
+    def get_compression_options(self):
+        return self.compression_options
 
     def on_close(self):
         self.close_future.set_result((self.close_code, self.close_reason))
@@ -45,6 +51,11 @@ class EchoHandler(TestWebSocketHandler):
         self.write_message(message, isinstance(message, bytes))
 
 
+class ErrorInOnMessageHandler(TestWebSocketHandler):
+    def on_message(self, message):
+        1/0
+
+
 class HeaderHandler(TestWebSocketHandler):
     def open(self):
         try:
@@ -67,7 +78,34 @@ class CloseReasonHandler(TestWebSocketHandler):
         self.close(1001, "goodbye")
 
 
-class WebSocketTest(AsyncHTTPTestCase):
+class AsyncPrepareHandler(TestWebSocketHandler):
+    @gen.coroutine
+    def prepare(self):
+        yield gen.moment
+
+    def on_message(self, message):
+        self.write_message(message)
+
+
+class WebSocketBaseTestCase(AsyncHTTPTestCase):
+    @gen.coroutine
+    def ws_connect(self, path, compression_options=None):
+        ws = yield websocket_connect(
+            'ws://localhost:%d%s' % (self.get_http_port(), path),
+            compression_options=compression_options)
+        raise gen.Return(ws)
+
+    @gen.coroutine
+    def close(self, ws):
+        """Close a websocket connection and wait for the server side.
+
+        If we don't wait here, there are sometimes leak warnings in the
+        tests.
+        """
+        ws.close()
+        yield self.close_future
+
+class WebSocketTest(WebSocketBaseTestCase):
     def get_app(self):
         self.close_future = Future()
         return Application([
@@ -76,6 +114,10 @@ class WebSocketTest(AsyncHTTPTestCase):
             ('/header', HeaderHandler, dict(close_future=self.close_future)),
             ('/close_reason', CloseReasonHandler,
              dict(close_future=self.close_future)),
+            ('/error_in_on_message', ErrorInOnMessageHandler,
+             dict(close_future=self.close_future)),
+            ('/async_prepare', AsyncPrepareHandler,
+             dict(close_future=self.close_future)),
         ])
 
     def test_http_request(self):
@@ -85,14 +127,11 @@ class WebSocketTest(AsyncHTTPTestCase):
 
     @gen_test
     def test_websocket_gen(self):
-        ws = yield websocket_connect(
+        ws = yield self.ws_connect('/echo')
-            'ws://localhost:%d/echo' % self.get_http_port(),
-            io_loop=self.io_loop)
         ws.write_message('hello')
         response = yield ws.read_message()
         self.assertEqual(response, 'hello')
-        ws.close()
+        yield self.close(ws)
-        yield self.close_future
 
     def test_websocket_callbacks(self):
         websocket_connect(
@@ -107,20 +146,41 @@ class WebSocketTest(AsyncHTTPTestCase):
         ws.close()
         self.wait()
 
+    @gen_test
+    def test_binary_message(self):
+        ws = yield self.ws_connect('/echo')
+        ws.write_message(b'hello \xe9', binary=True)
+        response = yield ws.read_message()
+        self.assertEqual(response, b'hello \xe9')
+        yield self.close(ws)
+
+    @gen_test
+    def test_unicode_message(self):
+        ws = yield self.ws_connect('/echo')
+        ws.write_message(u('hello \u00e9'))
+        response = yield ws.read_message()
+        self.assertEqual(response, u('hello \u00e9'))
+        yield self.close(ws)
+
+    @gen_test
+    def test_error_in_on_message(self):
+        ws = yield self.ws_connect('/error_in_on_message')
+        ws.write_message('hello')
+        with ExpectLog(app_log, "Uncaught exception"):
+            response = yield ws.read_message()
+        self.assertIs(response, None)
+        yield self.close(ws)
+
     @gen_test
     def test_websocket_http_fail(self):
         with self.assertRaises(HTTPError) as cm:
-            yield websocket_connect(
+            yield self.ws_connect('/notfound')
-                'ws://localhost:%d/notfound' % self.get_http_port(),
-                io_loop=self.io_loop)
         self.assertEqual(cm.exception.code, 404)
 
     @gen_test
     def test_websocket_http_success(self):
         with self.assertRaises(WebSocketError):
-            yield websocket_connect(
+            yield self.ws_connect('/non_ws')
-                'ws://localhost:%d/non_ws' % self.get_http_port(),
-                io_loop=self.io_loop)
 
     @gen_test
     def test_websocket_network_fail(self):
@@ -139,6 +199,7 @@ class WebSocketTest(AsyncHTTPTestCase):
             'ws://localhost:%d/echo' % self.get_http_port())
         ws.write_message('hello')
         ws.write_message('world')
+        # Close the underlying stream.
         ws.stream.close()
         yield self.close_future
 
@@ -150,13 +211,11 @@ class WebSocketTest(AsyncHTTPTestCase):
                         headers={'X-Test': 'hello'}))
         response = yield ws.read_message()
         self.assertEqual(response, 'hello')
-        ws.close()
+        yield self.close(ws)
-        yield self.close_future
 
     @gen_test
     def test_server_close_reason(self):
-        ws = yield websocket_connect(
+        ws = yield self.ws_connect('/close_reason')
-            'ws://localhost:%d/close_reason' % self.get_http_port())
         msg = yield ws.read_message()
         # A message of None means the other side closed the connection.
         self.assertIs(msg, None)
@@ -165,13 +224,21 @@ class WebSocketTest(AsyncHTTPTestCase):
 
     @gen_test
     def test_client_close_reason(self):
-        ws = yield websocket_connect(
+        ws = yield self.ws_connect('/echo')
-            'ws://localhost:%d/echo' % self.get_http_port())
         ws.close(1001, 'goodbye')
         code, reason = yield self.close_future
         self.assertEqual(code, 1001)
         self.assertEqual(reason, 'goodbye')
 
+    @gen_test
+    def test_async_prepare(self):
+        # Previously, an async prepare method triggered a bug that would
+        # result in a timeout on test shutdown (and a memory leak).
+        ws = yield self.ws_connect('/async_prepare')
+        ws.write_message('hello')
+        res = yield ws.read_message()
+        self.assertEqual(res, 'hello')
+
     @gen_test
     def test_check_origin_valid_no_path(self):
         port = self.get_http_port()
@@ -184,8 +251,7 @@ class WebSocketTest(AsyncHTTPTestCase):
         ws.write_message('hello')
         response = yield ws.read_message()
         self.assertEqual(response, 'hello')
-        ws.close()
+        yield self.close(ws)
-        yield self.close_future
 
     @gen_test
     def test_check_origin_valid_with_path(self):
@@ -199,8 +265,7 @@ class WebSocketTest(AsyncHTTPTestCase):
         ws.write_message('hello')
         response = yield ws.read_message()
         self.assertEqual(response, 'hello')
-        ws.close()
+        yield self.close(ws)
-        yield self.close_future
 
     @gen_test
     def test_check_origin_invalid_partial_url(self):
@@ -245,6 +310,78 @@ class WebSocketTest(AsyncHTTPTestCase):
         self.assertEqual(cm.exception.code, 403)
 
 
+class CompressionTestMixin(object):
+    MESSAGE = 'Hello world. Testing 123 123'
+
+    def get_app(self):
+        self.close_future = Future()
+        return Application([
+            ('/echo', EchoHandler, dict(
+                close_future=self.close_future,
+                compression_options=self.get_server_compression_options())),
+        ])
+
+    def get_server_compression_options(self):
+        return None
+
+    def get_client_compression_options(self):
+        return None
+
+    @gen_test
+    def test_message_sizes(self):
+        ws = yield self.ws_connect(
+            '/echo',
+            compression_options=self.get_client_compression_options())
+        # Send the same message three times so we can measure the
+        # effect of the context_takeover options.
+        for i in range(3):
+            ws.write_message(self.MESSAGE)
+            response = yield ws.read_message()
+            self.assertEqual(response, self.MESSAGE)
+        self.assertEqual(ws.protocol._message_bytes_out, len(self.MESSAGE) * 3)
+        self.assertEqual(ws.protocol._message_bytes_in, len(self.MESSAGE) * 3)
+        self.verify_wire_bytes(ws.protocol._wire_bytes_in,
+                               ws.protocol._wire_bytes_out)
+        yield self.close(ws)
+
+
+class UncompressedTestMixin(CompressionTestMixin):
+    """Specialization of CompressionTestMixin when we expect no compression."""
+    def verify_wire_bytes(self, bytes_in, bytes_out):
+        # Bytes out includes the 4-byte mask key per message.
+        self.assertEqual(bytes_out, 3 * (len(self.MESSAGE) + 6))
+        self.assertEqual(bytes_in, 3 * (len(self.MESSAGE) + 2))
+
+
+class NoCompressionTest(UncompressedTestMixin, WebSocketBaseTestCase):
+    pass
+
+
+# If only one side tries to compress, the extension is not negotiated.
+class ServerOnlyCompressionTest(UncompressedTestMixin, WebSocketBaseTestCase):
+    def get_server_compression_options(self):
+        return {}
+
+
+class ClientOnlyCompressionTest(UncompressedTestMixin, WebSocketBaseTestCase):
+    def get_client_compression_options(self):
+        return {}
+
+
+class DefaultCompressionTest(CompressionTestMixin, WebSocketBaseTestCase):
+    def get_server_compression_options(self):
+        return {}
+
+    def get_client_compression_options(self):
+        return {}
+
+    def verify_wire_bytes(self, bytes_in, bytes_out):
+        self.assertLess(bytes_out, 3 * (len(self.MESSAGE) + 6))
+        self.assertLess(bytes_in, 3 * (len(self.MESSAGE) + 2))
+        # Bytes out includes the 4 bytes mask key per message.
+        self.assertEqual(bytes_out, bytes_in + 12)
+
+
 class MaskFunctionMixin(object):
     # Subclasses should define self.mask(mask, data)
     def test_mask(self):

tornado/testing.py

@@ -28,7 +28,7 @@ except ImportError:
     IOLoop = None
     netutil = None
     SimpleAsyncHTTPClient = None
-from tornado.log import gen_log
+from tornado.log import gen_log, app_log
 from tornado.stack_context import ExceptionStackContext
 from tornado.util import raise_exc_info, basestring_type
 import functools
@@ -114,8 +114,8 @@ class _TestMethodWrapper(object):
     def __init__(self, orig_method):
         self.orig_method = orig_method
 
-    def __call__(self):
+    def __call__(self, *args, **kwargs):
-        result = self.orig_method()
+        result = self.orig_method(*args, **kwargs)
         if isinstance(result, types.GeneratorType):
             raise TypeError("Generator test methods should be decorated with "
                             "tornado.testing.gen_test")
@@ -237,7 +237,11 @@ class AsyncTestCase(unittest.TestCase):
         return IOLoop()
 
     def _handle_exception(self, typ, value, tb):
-        self.__failure = (typ, value, tb)
+        if self.__failure is None:
+            self.__failure = (typ, value, tb)
+        else:
+            app_log.error("multiple unhandled exceptions in test",
+                          exc_info=(typ, value, tb))
         self.stop()
         return True
 
@@ -395,7 +399,8 @@ class AsyncHTTPTestCase(AsyncTestCase):
 
     def tearDown(self):
         self.http_server.stop()
-        self.io_loop.run_sync(self.http_server.close_all_connections)
+        self.io_loop.run_sync(self.http_server.close_all_connections,
+                              timeout=get_async_test_timeout())
         if (not IOLoop.initialized() or
                 self.http_client.io_loop is not IOLoop.instance()):
             self.http_client.close()

tornado/util.py

@@ -115,16 +115,17 @@ def import_object(name):
 if type('') is not type(b''):
     def u(s):
         return s
-    bytes_type = bytes
     unicode_type = str
     basestring_type = str
 else:
     def u(s):
         return s.decode('unicode_escape')
-    bytes_type = str
     unicode_type = unicode
     basestring_type = basestring
 
+# Deprecated alias that was used before we dropped py25 support.
+# Left here in case anyone outside Tornado is using it.
+bytes_type = bytes
 
 if sys.version_info > (3,):
     exec("""
@@ -154,7 +155,7 @@ def errno_from_exception(e):
     """Provides the errno from an Exception object.
 
     There are cases that the errno attribute was not set so we pull
-    the errno out of the args but if someone instatiates an Exception
+    the errno out of the args but if someone instantiates an Exception
     without any args you will get a tuple error. So this function
     abstracts all that behavior to give you a safe way to get the
     errno.
@@ -202,7 +203,7 @@ class Configurable(object):
             impl = cls
         args.update(kwargs)
         instance = super(Configurable, cls).__new__(impl)
-        # initialize vs __init__ chosen for compatiblity with AsyncHTTPClient
+        # initialize vs __init__ chosen for compatibility with AsyncHTTPClient
         # singleton magic.  If we get rid of that we can switch to __init__
         # here too.
         instance.initialize(**args)
@@ -237,7 +238,7 @@ class Configurable(object):
         some parameters.
         """
         base = cls.configurable_base()
-        if isinstance(impl, (unicode_type, bytes_type)):
+        if isinstance(impl, (unicode_type, bytes)):
             impl = import_object(impl)
         if impl is not None and not issubclass(impl, cls):
             raise ValueError("Invalid subclass of %s" % cls)

tornado/web.py

@@ -35,8 +35,7 @@ Here is a simple "Hello, world" example app::
         application.listen(8888)
         tornado.ioloop.IOLoop.instance().start()
 
-See the :doc:`Tornado overview <overview>` for more details and a good getting
+See the :doc:`guide` for additional information.
-started guide.
 
 Thread-safety notes
 -------------------
@@ -48,6 +47,7 @@ not thread-safe.  In particular, methods such as
 you use multiple threads it is important to use `.IOLoop.add_callback`
 to transfer control back to the main thread before finishing the
 request.
+
 """
 
 from __future__ import absolute_import, division, print_function, with_statement
@@ -72,6 +72,7 @@ import time
 import tornado
 import traceback
 import types
+from io import BytesIO
 
 from tornado.concurrent import Future, is_future
 from tornado import escape
@@ -83,12 +84,8 @@ from tornado.log import access_log, app_log, gen_log
 from tornado import stack_context
 from tornado import template
 from tornado.escape import utf8, _unicode
-from tornado.util import bytes_type, import_object, ObjectDict, raise_exc_info, unicode_type, _websocket_mask
+from tornado.util import import_object, ObjectDict, raise_exc_info, unicode_type, _websocket_mask
 
-try:
-    from io import BytesIO  # python 3
-except ImportError:
-    from cStringIO import StringIO as BytesIO  # python 2
 
 try:
     import Cookie  # py2
@@ -344,7 +341,7 @@ class RequestHandler(object):
     _INVALID_HEADER_CHAR_RE = re.compile(br"[\x00-\x1f]")
 
     def _convert_header_value(self, value):
-        if isinstance(value, bytes_type):
+        if isinstance(value, bytes):
             pass
         elif isinstance(value, unicode_type):
             value = value.encode('utf-8')
@@ -652,7 +649,7 @@ class RequestHandler(object):
             raise RuntimeError("Cannot write() after finish().  May be caused "
                                "by using async operations without the "
                                "@asynchronous decorator.")
-        if not isinstance(chunk, (bytes_type, unicode_type, dict)):
+        if not isinstance(chunk, (bytes, unicode_type, dict)):
             raise TypeError("write() only accepts bytes, unicode, and dict objects")
         if isinstance(chunk, dict):
             chunk = escape.json_encode(chunk)
@@ -677,7 +674,7 @@ class RequestHandler(object):
                 js_embed.append(utf8(embed_part))
             file_part = module.javascript_files()
             if file_part:
-                if isinstance(file_part, (unicode_type, bytes_type)):
+                if isinstance(file_part, (unicode_type, bytes)):
                     js_files.append(file_part)
                 else:
                     js_files.extend(file_part)
@@ -686,7 +683,7 @@ class RequestHandler(object):
                 css_embed.append(utf8(embed_part))
             file_part = module.css_files()
             if file_part:
-                if isinstance(file_part, (unicode_type, bytes_type)):
+                if isinstance(file_part, (unicode_type, bytes)):
                     css_files.append(file_part)
                 else:
                     css_files.extend(file_part)
@@ -919,7 +916,7 @@ class RequestHandler(object):
             return
         self.clear()
 
-        reason = None
+        reason = kwargs.get('reason')
         if 'exc_info' in kwargs:
             exception = kwargs['exc_info'][1]
             if isinstance(exception, HTTPError) and exception.reason:
@@ -959,12 +956,15 @@ class RequestHandler(object):
 
     @property
     def locale(self):
-        """The local for the current session.
+        """The locale for the current session.
 
         Determined by either `get_user_locale`, which you can override to
         set the locale based on, e.g., a user preference stored in a
         database, or `get_browser_locale`, which uses the ``Accept-Language``
         header.
+
+        .. versionchanged: 4.1
+           Added a property setter.
         """
         if not hasattr(self, "_locale"):
             self._locale = self.get_user_locale()
@@ -973,6 +973,10 @@ class RequestHandler(object):
                 assert self._locale
         return self._locale
 
+    @locale.setter
+    def locale(self, value):
+        self._locale = value
+
     def get_user_locale(self):
         """Override to determine the locale from the authenticated user.
 
@@ -1128,14 +1132,15 @@ class RequestHandler(object):
             else:
                 # Treat unknown versions as not present instead of failing.
                 return None, None, None
-        elif len(cookie) == 32:
+        else:
             version = 1
-            token = binascii.a2b_hex(utf8(cookie))
+            try:
+                token = binascii.a2b_hex(utf8(cookie))
+            except (binascii.Error, TypeError):
+                token = utf8(cookie)
             # We don't have a usable timestamp in older versions.
             timestamp = int(time.time())
             return (version, token, timestamp)
-        else:
-            return None, None, None
 
     def check_xsrf_cookie(self):
         """Verifies that the ``_xsrf`` cookie matches the ``_xsrf`` argument.
@@ -1627,7 +1632,7 @@ class Application(httputil.HTTPServerConnectionDelegate):
                  **settings):
         if transforms is None:
             self.transforms = []
-            if settings.get("gzip"):
+            if settings.get("compress_response") or settings.get("gzip"):
                 self.transforms.append(GZipContentEncoding)
         else:
             self.transforms = transforms
@@ -2164,11 +2169,14 @@ class StaticFileHandler(RequestHandler):
 
         if include_body:
             content = self.get_content(self.absolute_path, start, end)
-            if isinstance(content, bytes_type):
+            if isinstance(content, bytes):
                 content = [content]
             for chunk in content:
-                self.write(chunk)
+                try:
-                yield self.flush()
+                    self.write(chunk)
+                    yield self.flush()
+                except iostream.StreamClosedError:
+                    return
         else:
             assert self.request.method == "HEAD"
 
@@ -2335,7 +2343,7 @@ class StaticFileHandler(RequestHandler):
         """
         data = cls.get_content(abspath)
         hasher = hashlib.md5()
-        if isinstance(data, bytes_type):
+        if isinstance(data, bytes):
             hasher.update(data)
         else:
             for chunk in data:
@@ -2547,7 +2555,6 @@ class GZipContentEncoding(OutputTransform):
             ctype = _unicode(headers.get("Content-Type", "")).split(";")[0]
             self._gzipping = self._compressible_type(ctype) and \
                 (not finishing or len(chunk) >= self.MIN_LENGTH) and \
-                (finishing or "Content-Length" not in headers) and \
                 ("Content-Encoding" not in headers)
         if self._gzipping:
             headers["Content-Encoding"] = "gzip"
@@ -2555,7 +2562,14 @@ class GZipContentEncoding(OutputTransform):
             self._gzip_file = gzip.GzipFile(mode="w", fileobj=self._gzip_value)
             chunk = self.transform_chunk(chunk, finishing)
             if "Content-Length" in headers:
-                headers["Content-Length"] = str(len(chunk))
+                # The original content length is no longer correct.
+                # If this is the last (and only) chunk, we can set the new
+                # content-length; otherwise we remove it and fall back to
+                # chunked encoding.
+                if finishing:
+                    headers["Content-Length"] = str(len(chunk))
+                else:
+                    del headers["Content-Length"]
         return status_code, headers, chunk
 
     def transform_chunk(self, chunk, finishing):
@@ -2704,7 +2718,7 @@ class TemplateModule(UIModule):
     def javascript_files(self):
         result = []
         for f in self._get_resources("javascript_files"):
-            if isinstance(f, (unicode_type, bytes_type)):
+            if isinstance(f, (unicode_type, bytes)):
                 result.append(f)
             else:
                 result.extend(f)
@@ -2716,7 +2730,7 @@ class TemplateModule(UIModule):
     def css_files(self):
         result = []
         for f in self._get_resources("css_files"):
-            if isinstance(f, (unicode_type, bytes_type)):
+            if isinstance(f, (unicode_type, bytes)):
                 result.append(f)
             else:
                 result.extend(f)
@@ -2754,7 +2768,7 @@ class URLSpec(object):
           in the regex will be passed in to the handler's get/post/etc
           methods as arguments.
 
-        * ``handler_class``: `RequestHandler` subclass to be invoked.
+        * ``handler``: `RequestHandler` subclass to be invoked.
 
         * ``kwargs`` (optional): A dictionary of additional arguments
           to be passed to the handler's constructor.
@@ -2821,7 +2835,7 @@ class URLSpec(object):
             return self._path
         converted_args = []
         for a in args:
-            if not isinstance(a, (unicode_type, bytes_type)):
+            if not isinstance(a, (unicode_type, bytes)):
                 a = str(a)
             converted_args.append(escape.url_escape(utf8(a), plus=False))
         return self._path % tuple(converted_args)

tornado/websocket.py

@@ -26,6 +26,7 @@ import os
 import struct
 import tornado.escape
 import tornado.web
+import zlib
 
 from tornado.concurrent import TracebackFuture
 from tornado.escape import utf8, native_str, to_unicode
@@ -35,7 +36,7 @@ from tornado.iostream import StreamClosedError
 from tornado.log import gen_log, app_log
 from tornado import simple_httpclient
 from tornado.tcpclient import TCPClient
-from tornado.util import bytes_type, _websocket_mask
+from tornado.util import _websocket_mask
 
 try:
     from urllib.parse import urlparse # py2
@@ -105,6 +106,21 @@ class WebSocketHandler(tornado.web.RequestHandler):
       };
 
     This script pops up an alert box that says "You said: Hello, world".
+
+    Web browsers allow any site to open a websocket connection to any other,
+    instead of using the same-origin policy that governs other network
+    access from javascript.  This can be surprising and is a potential
+    security hole, so since Tornado 4.0 `WebSocketHandler` requires
+    applications that wish to receive cross-origin websockets to opt in
+    by overriding the `~WebSocketHandler.check_origin` method (see that
+    method's docs for details).  Failure to do so is the most likely
+    cause of 403 errors when making a websocket connection.
+
+    When using a secure websocket connection (``wss://``) with a self-signed
+    certificate, the connection from a browser may fail because it wants
+    to show the "accept this certificate" dialog but has nowhere to show it.
+    You must first visit a regular HTML page using the same certificate
+    to accept it before the websocket connection will succeed.
     """
     def __init__(self, application, request, **kwargs):
         tornado.web.RequestHandler.__init__(self, application, request,
@@ -156,13 +172,15 @@ class WebSocketHandler(tornado.web.RequestHandler):
         self.stream.set_close_callback(self.on_connection_close)
 
         if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8", "13"):
-            self.ws_connection = WebSocketProtocol13(self)
+            self.ws_connection = WebSocketProtocol13(
+                self, compression_options=self.get_compression_options())
             self.ws_connection.accept_connection()
         else:
-            self.stream.write(tornado.escape.utf8(
+            if not self.stream.closed():
-                "HTTP/1.1 426 Upgrade Required\r\n"
+                self.stream.write(tornado.escape.utf8(
-                "Sec-WebSocket-Version: 8\r\n\r\n"))
+                    "HTTP/1.1 426 Upgrade Required\r\n"
-            self.stream.close()
+                    "Sec-WebSocket-Version: 8\r\n\r\n"))
+                self.stream.close()
 
 
     def write_message(self, message, binary=False):
@@ -198,6 +216,19 @@ class WebSocketHandler(tornado.web.RequestHandler):
         """
         return None
 
+    def get_compression_options(self):
+        """Override to return compression options for the connection.
+
+        If this method returns None (the default), compression will
+        be disabled.  If it returns a dict (even an empty one), it
+        will be enabled.  The contents of the dict may be used to
+        control the memory and CPU usage of the compression,
+        but no such options are currently implemented.
+
+        .. versionadded:: 4.1
+        """
+        return None
+
     def open(self):
         """Invoked when a new WebSocket is opened.
 
@@ -275,6 +306,19 @@ class WebSocketHandler(tornado.web.RequestHandler):
         browsers, since WebSockets are allowed to bypass the usual same-origin
         policies and don't use CORS headers.
 
+        To accept all cross-origin traffic (which was the default prior to
+        Tornado 4.0), simply override this method to always return true::
+
+            def check_origin(self, origin):
+                return True
+
+        To allow connections from any subdomain of your site, you might
+        do something like::
+
+            def check_origin(self, origin):
+                parsed_origin = urllib.parse.urlparse(origin)
+                return parsed_origin.netloc.endswith(".mydomain.com")
+
         .. versionadded:: 4.0
         """
         parsed_origin = urlparse(origin)
@@ -308,6 +352,15 @@ class WebSocketHandler(tornado.web.RequestHandler):
             self.ws_connection = None
             self.on_close()
 
+    def send_error(self, *args, **kwargs):
+        if self.stream is None:
+            super(WebSocketHandler, self).send_error(*args, **kwargs)
+        else:
+            # If we get an uncaught exception during the handshake,
+            # we have no choice but to abruptly close the connection.
+            # TODO: for uncaught exceptions after the handshake,
+            # we can close the connection more gracefully.
+            self.stream.close()
 
 def _wrap_method(method):
     def _disallow_for_websocket(self, *args, **kwargs):
@@ -316,7 +369,7 @@ def _wrap_method(method):
         else:
             raise RuntimeError("Method not supported for Web Sockets")
     return _disallow_for_websocket
-for method in ["write", "redirect", "set_header", "send_error", "set_cookie",
+for method in ["write", "redirect", "set_header", "set_cookie",
                "set_status", "flush", "finish"]:
     setattr(WebSocketHandler, method,
             _wrap_method(getattr(WebSocketHandler, method)))
@@ -355,13 +408,68 @@ class WebSocketProtocol(object):
         self.close()  # let the subclass cleanup
 
 
+class _PerMessageDeflateCompressor(object):
+    def __init__(self, persistent, max_wbits):
+        if max_wbits is None:
+            max_wbits = zlib.MAX_WBITS
+        # There is no symbolic constant for the minimum wbits value.
+        if not (8 <= max_wbits <= zlib.MAX_WBITS):
+            raise ValueError("Invalid max_wbits value %r; allowed range 8-%d",
+                             max_wbits, zlib.MAX_WBITS)
+        self._max_wbits = max_wbits
+        if persistent:
+            self._compressor = self._create_compressor()
+        else:
+            self._compressor = None
+
+    def _create_compressor(self):
+        return zlib.compressobj(-1, zlib.DEFLATED, -self._max_wbits)
+
+    def compress(self, data):
+        compressor = self._compressor or self._create_compressor()
+        data = (compressor.compress(data) +
+                compressor.flush(zlib.Z_SYNC_FLUSH))
+        assert data.endswith(b'\x00\x00\xff\xff')
+        return data[:-4]
+
+
+class _PerMessageDeflateDecompressor(object):
+    def __init__(self, persistent, max_wbits):
+        if max_wbits is None:
+            max_wbits = zlib.MAX_WBITS
+        if not (8 <= max_wbits <= zlib.MAX_WBITS):
+            raise ValueError("Invalid max_wbits value %r; allowed range 8-%d",
+                             max_wbits, zlib.MAX_WBITS)
+        self._max_wbits = max_wbits
+        if persistent:
+            self._decompressor = self._create_decompressor()
+        else:
+            self._decompressor = None
+
+    def _create_decompressor(self):
+        return zlib.decompressobj(-self._max_wbits)
+
+    def decompress(self, data):
+        decompressor = self._decompressor or self._create_decompressor()
+        return decompressor.decompress(data + b'\x00\x00\xff\xff')
+
+
 class WebSocketProtocol13(WebSocketProtocol):
     """Implementation of the WebSocket protocol from RFC 6455.
 
     This class supports versions 7 and 8 of the protocol in addition to the
     final version 13.
     """
-    def __init__(self, handler, mask_outgoing=False):
+    # Bit masks for the first byte of a frame.
+    FIN = 0x80
+    RSV1 = 0x40
+    RSV2 = 0x20
+    RSV3 = 0x10
+    RSV_MASK = RSV1 | RSV2 | RSV3
+    OPCODE_MASK = 0x0f
+
+    def __init__(self, handler, mask_outgoing=False,
+                 compression_options=None):
         WebSocketProtocol.__init__(self, handler)
         self.mask_outgoing = mask_outgoing
         self._final_frame = False
@@ -372,6 +480,19 @@ class WebSocketProtocol13(WebSocketProtocol):
         self._fragmented_message_buffer = None
         self._fragmented_message_opcode = None
         self._waiting = None
+        self._compression_options = compression_options
+        self._decompressor = None
+        self._compressor = None
+        self._frame_compressed = None
+        # The total uncompressed size of all messages received or sent.
+        # Unicode messages are encoded to utf8.
+        # Only for testing; subject to change.
+        self._message_bytes_in = 0
+        self._message_bytes_out = 0
+        # The total size of all packets received or sent.  Includes
+        # the effect of compression, frame overhead, and control frames.
+        self._wire_bytes_in = 0
+        self._wire_bytes_out = 0
 
     def accept_connection(self):
         try:
@@ -416,24 +537,99 @@ class WebSocketProtocol13(WebSocketProtocol):
                 assert selected in subprotocols
                 subprotocol_header = "Sec-WebSocket-Protocol: %s\r\n" % selected
 
+        extension_header = ''
+        extensions = self._parse_extensions_header(self.request.headers)
+        for ext in extensions:
+            if (ext[0] == 'permessage-deflate' and
+                self._compression_options is not None):
+                # TODO: negotiate parameters if compression_options
+                # specifies limits.
+                self._create_compressors('server', ext[1])
+                if ('client_max_window_bits' in ext[1] and
+                    ext[1]['client_max_window_bits'] is None):
+                    # Don't echo an offered client_max_window_bits
+                    # parameter with no value.
+                    del ext[1]['client_max_window_bits']
+                extension_header = ('Sec-WebSocket-Extensions: %s\r\n' %
+                                    httputil._encode_header(
+                                        'permessage-deflate', ext[1]))
+                break
+
+        if self.stream.closed():
+            self._abort()
+            return
         self.stream.write(tornado.escape.utf8(
             "HTTP/1.1 101 Switching Protocols\r\n"
             "Upgrade: websocket\r\n"
             "Connection: Upgrade\r\n"
             "Sec-WebSocket-Accept: %s\r\n"
-            "%s"
+            "%s%s"
-            "\r\n" % (self._challenge_response(), subprotocol_header)))
+            "\r\n" % (self._challenge_response(),
+                      subprotocol_header, extension_header)))
 
         self._run_callback(self.handler.open, *self.handler.open_args,
                            **self.handler.open_kwargs)
         self._receive_frame()
 
-    def _write_frame(self, fin, opcode, data):
+    def _parse_extensions_header(self, headers):
+        extensions = headers.get("Sec-WebSocket-Extensions", '')
+        if extensions:
+            return [httputil._parse_header(e.strip())
+                    for e in extensions.split(',')]
+        return []
+
+    def _process_server_headers(self, key, headers):
+        """Process the headers sent by the server to this client connection.
+
+        'key' is the websocket handshake challenge/response key.
+        """
+        assert headers['Upgrade'].lower() == 'websocket'
+        assert headers['Connection'].lower() == 'upgrade'
+        accept = self.compute_accept_value(key)
+        assert headers['Sec-Websocket-Accept'] == accept
+
+        extensions = self._parse_extensions_header(headers)
+        for ext in extensions:
+            if (ext[0] == 'permessage-deflate' and
+                self._compression_options is not None):
+                self._create_compressors('client', ext[1])
+            else:
+                raise ValueError("unsupported extension %r", ext)
+
+    def _get_compressor_options(self, side, agreed_parameters):
+        """Converts a websocket agreed_parameters set to keyword arguments
+        for our compressor objects.
+        """
+        options = dict(
+            persistent=(side + '_no_context_takeover') not in agreed_parameters)
+        wbits_header = agreed_parameters.get(side + '_max_window_bits', None)
+        if wbits_header is None:
+            options['max_wbits'] = zlib.MAX_WBITS
+        else:
+            options['max_wbits'] = int(wbits_header)
+        return options
+
+    def _create_compressors(self, side, agreed_parameters):
+        # TODO: handle invalid parameters gracefully
+        allowed_keys = set(['server_no_context_takeover',
+                            'client_no_context_takeover',
+                            'server_max_window_bits',
+                            'client_max_window_bits'])
+        for key in agreed_parameters:
+            if key not in allowed_keys:
+                raise ValueError("unsupported compression parameter %r" % key)
+        other_side = 'client' if (side == 'server') else 'server'
+        self._compressor = _PerMessageDeflateCompressor(
+            **self._get_compressor_options(side, agreed_parameters))
+        self._decompressor = _PerMessageDeflateDecompressor(
+            **self._get_compressor_options(other_side, agreed_parameters))
+
+    def _write_frame(self, fin, opcode, data, flags=0):
         if fin:
-            finbit = 0x80
+            finbit = self.FIN
         else:
             finbit = 0
-        frame = struct.pack("B", finbit | opcode)
+        frame = struct.pack("B", finbit | opcode | flags)
         l = len(data)
         if self.mask_outgoing:
             mask_bit = 0x80
@@ -449,7 +645,11 @@ class WebSocketProtocol13(WebSocketProtocol):
             mask = os.urandom(4)
             data = mask + _websocket_mask(mask, data)
         frame += data
-        self.stream.write(frame)
+        self._wire_bytes_out += len(frame)
+        try:
+            self.stream.write(frame)
+        except StreamClosedError:
+            self._abort()
 
     def write_message(self, message, binary=False):
         """Sends the given message to the client of this Web Socket."""
@@ -458,15 +658,17 @@ class WebSocketProtocol13(WebSocketProtocol):
         else:
             opcode = 0x1
         message = tornado.escape.utf8(message)
-        assert isinstance(message, bytes_type)
+        assert isinstance(message, bytes)
-        try:
+        self._message_bytes_out += len(message)
-            self._write_frame(True, opcode, message)
+        flags = 0
-        except StreamClosedError:
+        if self._compressor:
-            self._abort()
+            message = self._compressor.compress(message)
+            flags |= self.RSV1
+        self._write_frame(True, opcode, message, flags=flags)
 
     def write_ping(self, data):
         """Send ping frame."""
-        assert isinstance(data, bytes_type)
+        assert isinstance(data, bytes)
         self._write_frame(True, 0x9, data)
 
     def _receive_frame(self):
@@ -476,11 +678,15 @@ class WebSocketProtocol13(WebSocketProtocol):
             self._abort()
 
     def _on_frame_start(self, data):
+        self._wire_bytes_in += len(data)
         header, payloadlen = struct.unpack("BB", data)
-        self._final_frame = header & 0x80
+        self._final_frame = header & self.FIN
-        reserved_bits = header & 0x70
+        reserved_bits = header & self.RSV_MASK
-        self._frame_opcode = header & 0xf
+        self._frame_opcode = header & self.OPCODE_MASK
         self._frame_opcode_is_control = self._frame_opcode & 0x8
+        if self._decompressor is not None:
+            self._frame_compressed = bool(reserved_bits & self.RSV1)
+            reserved_bits &= ~self.RSV1
         if reserved_bits:
             # client is using as-yet-undefined extensions; abort
             self._abort()
@@ -506,6 +712,7 @@ class WebSocketProtocol13(WebSocketProtocol):
             self._abort()
 
     def _on_frame_length_16(self, data):
+        self._wire_bytes_in += len(data)
         self._frame_length = struct.unpack("!H", data)[0]
         try:
             if self._masked_frame:
@@ -516,6 +723,7 @@ class WebSocketProtocol13(WebSocketProtocol):
             self._abort()
 
     def _on_frame_length_64(self, data):
+        self._wire_bytes_in += len(data)
         self._frame_length = struct.unpack("!Q", data)[0]
         try:
             if self._masked_frame:
@@ -526,6 +734,7 @@ class WebSocketProtocol13(WebSocketProtocol):
             self._abort()
 
     def _on_masking_key(self, data):
+        self._wire_bytes_in += len(data)
         self._frame_mask = data
         try:
             self.stream.read_bytes(self._frame_length, self._on_masked_frame_data)
@@ -533,9 +742,11 @@ class WebSocketProtocol13(WebSocketProtocol):
             self._abort()
 
     def _on_masked_frame_data(self, data):
+        # Don't touch _wire_bytes_in; we'll do it in _on_frame_data.
         self._on_frame_data(_websocket_mask(self._frame_mask, data))
 
     def _on_frame_data(self, data):
+        self._wire_bytes_in += len(data)
         if self._frame_opcode_is_control:
             # control frames may be interleaved with a series of fragmented
             # data frames, so control frames must not interact with
@@ -576,8 +787,12 @@ class WebSocketProtocol13(WebSocketProtocol):
         if self.client_terminated:
             return
 
+        if self._frame_compressed:
+            data = self._decompressor.decompress(data)
+
         if opcode == 0x1:
             # UTF-8 data
+            self._message_bytes_in += len(data)
             try:
                 decoded = data.decode("utf-8")
             except UnicodeDecodeError:
@@ -586,7 +801,8 @@ class WebSocketProtocol13(WebSocketProtocol):
             self._run_callback(self.handler.on_message, decoded)
         elif opcode == 0x2:
             # Binary data
-            self._run_callback(self.handler.on_message, decoded)
+            self._message_bytes_in += len(data)
+            self._run_callback(self.handler.on_message, data)
         elif opcode == 0x8:
             # Close
             self.client_terminated = True
@@ -636,7 +852,8 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
     This class should not be instantiated directly; use the
     `websocket_connect` function instead.
     """
-    def __init__(self, io_loop, request):
+    def __init__(self, io_loop, request, compression_options=None):
+        self.compression_options = compression_options
         self.connect_future = TracebackFuture()
         self.read_future = None
         self.read_queue = collections.deque()
@@ -651,6 +868,14 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
             'Sec-WebSocket-Key': self.key,
             'Sec-WebSocket-Version': '13',
         })
+        if self.compression_options is not None:
+            # Always offer to let the server set our max_wbits (and even though
+            # we don't offer it, we will accept a client_no_context_takeover
+            # from the server).
+            # TODO: set server parameters for deflate extension
+            # if requested in self.compression_options.
+            request.headers['Sec-WebSocket-Extensions'] = (
+                'permessage-deflate; client_max_window_bits')
 
         self.tcp_client = TCPClient(io_loop=io_loop)
         super(WebSocketClientConnection, self).__init__(
@@ -673,10 +898,12 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
             self.protocol.close(code, reason)
             self.protocol = None
 
-    def _on_close(self):
+    def on_connection_close(self):
+        if not self.connect_future.done():
+            self.connect_future.set_exception(StreamClosedError())
         self.on_message(None)
-        self.resolver.close()
+        self.tcp_client.close()
-        super(WebSocketClientConnection, self)._on_close()
+        super(WebSocketClientConnection, self).on_connection_close()
 
     def _on_http_response(self, response):
         if not self.connect_future.done():
@@ -692,12 +919,10 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
                 start_line, headers)
 
         self.headers = headers
-        assert self.headers['Upgrade'].lower() == 'websocket'
+        self.protocol = WebSocketProtocol13(
-        assert self.headers['Connection'].lower() == 'upgrade'
+            self, mask_outgoing=True,
-        accept = WebSocketProtocol13.compute_accept_value(self.key)
+            compression_options=self.compression_options)
-        assert self.headers['Sec-Websocket-Accept'] == accept
+        self.protocol._process_server_headers(self.key, self.headers)
-
-        self.protocol = WebSocketProtocol13(self, mask_outgoing=True)
         self.protocol._receive_frame()
 
         if self._timeout is not None:
@@ -705,7 +930,12 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
             self._timeout = None
 
         self.stream = self.connection.detach()
-        self.stream.set_close_callback(self._on_close)
+        self.stream.set_close_callback(self.on_connection_close)
+        # Once we've taken over the connection, clear the final callback
+        # we set on the http request.  This deactivates the error handling
+        # in simple_httpclient that would otherwise interfere with our
+        # ability to see exceptions.
+        self.final_callback = None
 
         self.connect_future.set_result(self)
 
@@ -742,14 +972,21 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
         pass
 
 
-def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None):
+def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None,
+                      compression_options=None):
     """Client-side websocket support.
 
     Takes a url and returns a Future whose result is a
     `WebSocketClientConnection`.
 
+    ``compression_options`` is interpreted in the same way as the
+    return value of `.WebSocketHandler.get_compression_options`.
+
     .. versionchanged:: 3.2
        Also accepts ``HTTPRequest`` objects in place of urls.
+
+    .. versionchanged:: 4.1
+       Added ``compression_options``.
     """
     if io_loop is None:
         io_loop = IOLoop.current()
@@ -763,7 +1000,7 @@ def websocket_connect(url, io_loop=None, callback=None, connect_timeout=None):
         request = httpclient.HTTPRequest(url, connect_timeout=connect_timeout)
     request = httpclient._RequestProxy(
         request, httpclient.HTTPRequest._DEFAULTS)
-    conn = WebSocketClientConnection(io_loop, request)
+    conn = WebSocketClientConnection(io_loop, request, compression_options)
     if callback is not None:
         io_loop.add_future(conn.connect_future, callback)
     return conn.connect_future

tornado/wsgi.py

@@ -32,6 +32,7 @@ provides WSGI support in two ways:
 from __future__ import absolute_import, division, print_function, with_statement
 
 import sys
+from io import BytesIO
 import tornado
 
 from tornado.concurrent import Future
@@ -40,12 +41,8 @@ from tornado import httputil
 from tornado.log import access_log
 from tornado import web
 from tornado.escape import native_str
-from tornado.util import bytes_type, unicode_type
+from tornado.util import unicode_type
 
-try:
-    from io import BytesIO  # python 3
-except ImportError:
-    from cStringIO import StringIO as BytesIO  # python 2
 
 try:
     import urllib.parse as urllib_parse  # py3
@@ -58,7 +55,7 @@ except ImportError:
 # here to minimize the temptation to use them in non-wsgi contexts.
 if str is unicode_type:
     def to_wsgi_str(s):
-        assert isinstance(s, bytes_type)
+        assert isinstance(s, bytes)
         return s.decode('latin1')
 
     def from_wsgi_str(s):
@@ -66,7 +63,7 @@ if str is unicode_type:
         return s.encode('latin1')
 else:
     def to_wsgi_str(s):
-        assert isinstance(s, bytes_type)
+        assert isinstance(s, bytes)
         return s
 
     def from_wsgi_str(s):