October 12 2010
By Bill Krom
In: Tutorials - Leave a Comment
So I was looking over the Twitter API the other day, and I noticed they have officially switched their system to the Oauth protocol. All 3rd party applications now need to go through a complicated (while better) hand shake process before being granted access to account information. This is a giant leap forward for Twitter, but it may leave most average web developers scratching their heads, breaking things, and even slapping the keyboard around a bit and no one wants to see that!
I have written an application that will pull your twitter feed from twitter via access to your publicly available RSS feed, slide the data into your own database, and run quietly in the background all while remaining within the usage requirements set forth by Twitter.
I had no need to collect every bit of information from the BizLab Twitter account. I merely needed a way to display a "live" feed of recent tweets for the visitors to our website. In the anticipation of web designers and developers that will not be interested in creating complex applications that work with the new Oauth system, I integrated more functionality to allow the system to easily adapt to various needs.
This application will:
<div id="twitter_feed">
<div id="tw_header">
<p><a href="http://www.twitter.com/bizlabweb" target="_blank">Follow BizLab On Twitter</a></p>
</div><!-- end of asset_header div -->
<div id="tw_content">
<div id="tw_feed_data">
<!-- AjAX tweet content -->
<!-- this content is replaced with the twitter feed -->
<div class="inner_tw_box">
<p>Content item 1</p>
</div>
<div class="inner_tw_box">
<p>Content item 1</p>
</div>
<div class="inner_tw_box">
<p>Content item 1</p>
</div>
<div class="inner_tw_box">
<p>Content item 1</p>
</div>
<div class="inner_tw_box">
<p>Content item 1</p>
</div>
</div><!-- end of tw_feed_data div -->
</div><!-- end of tw_content div -->
<div class="more"><p><a href="http://www.twitter.com/bizlabweb" target="_blank">Follow Us</a></p></div>
</div><!-- end of twitter_feed div -->
<?php
require_once('../globals/appvars.php');
// initialize the array
$tweet_stack = array();
// get the current tweets
$query = "SELECT content, pub_date, DATE_FORMAT(pub_date, '%M %d at %l:%i%p') AS pd FROM tweets ORDER BY pub_date DESC";
$result = $db->query($query);
if($result->num_rows > 0){
while($row = $result->fetch_assoc()){
$content = htmlentities(stripslashes($row['content']));
$pub_date = $row['pd'];
// SEND THIS BACK AS JSON DATA
$new_tweet = array('content'=>$content, 'pub_date'=>$pub_date);
// push this item into the tweet_stack array
array_push($tweet_stack, $new_tweet);
}
// send JSON data back to the AJAX request
echo json_encode($tweet_stack);
}
else{
return false;
}
?>
function orderTweets($x, $y){
$x = strtotime($x['pub_date']);
$y = strtotime($y['pub_date']);
if($x==$y){
return 0;
}
else if($x<$y){
return 1;
}
else{
return -1;
}
}
function checkRecord($content, $pub_date){
global $db;
$query = "SELECT * FROM tweets WHERE content='$content' AND pub_date='$pub_date'";
$result = $db->query($query);
if($result->num_rows==0){
return false;
}
else{
return true;
}
}
function insertTweet($content, $pub_date){
// the current tweet is not in the DB, so insert it now
global $db;
$query = "INSERT INTO tweets (content, pub_date) VALUES('$content','$pub_date')";
$db->query($query);
}
function deleteTweet(){
global $db;
// find the oldest tweet record
$query = "SELECT MIN(pub_date) AS pub_date FROM tweets";
$result = $db->query($query);
$row = $result->fetch_assoc();
$pub_date = $row['pub_date'];
// now delete this oldest tweet
$q2 = "DELETE FROM tweets WHERE pub_date='$pub_date' LIMIT 1";
$db->query($q2);
}
function updateDB($tweet_stack){
// get the DB records
global $db;
$query = "SELECT content, pub_date FROM tweets ORDER BY pub_date DESC";
$results = $db->query($query);
while($row = $results->fetch_assoc()){
$current_pub_date = $row['pub_date'];
if(array_search($current_pub_date, $tweet_stack)===false){
// this tweet has been deleted on Twitter, delete it in the DB
$q2 = "DELETE FROM tweets WHERE pub_date='$current_pub_date' LIMIT 1";
$db->query($q2);
$i=0; // set the iterator for the foreach loop.
foreach($tweet_stack as $stack){
if($i>0){ break; }
$content = mysqli_real_escape_string($db, $stack['content']);
$pub_date = date("Y-m-d H:i:s",strtotime($stack['pub_date']));
// check to see if this tweet has already been recorded
$existing = checkRecord($content, $pub_date);
// insert the newest tweet that isn't already in the DB
if(!$existing){
insertTweet($content, $pub_date);
$i++;
}
}
}
}
}
// set your email address here
$email = 'bill@bizlab.us';
// set this link to your personal twitter RSS feed
$url = 'http://twitter.com/statuses/user_timeline/172689882.rss';
// set your tweet record limit
// if you change this number, you will need to modify both the CSS and Javascript controlling the scroll distance
$tweet_record_limit = 5;
// initialize the array
$tweet_stack = array();
// begin your cURL request
$tw = curl_init();
curl_setopt($tw, CURLOPT_URL, $url);
curl_setopt($tw, CURLOPT_HEADER, false); // turns off HTTP headers with output
curl_setopt($tw, CURLOPT_RETURNTRANSFER, TRUE);
$data = curl_exec($tw);
curl_close($tw);
if((strpos($data_string,'over capacity')===false)
&&(strpos($data_string, 'rate limit exceeded')===false)
&&(!empty($data_string))){
if($my_var==50){
// true or majority result
}
else{
// on failure, or alternate condition
}
// begin manipulating the ATOM FEED
$tweets = new SimpleXMLElement($data);
// loop through the array to get the tweets
foreach($tweets->channel->item as $tweet){
$published = $tweet->pubDate; // RSS DATE FORMAT: Thu, 05 Aug 2010 11:24:35 +0000
$content = trim($tweet->description);
// push data into an array
$new_tweet = array('content'=>$content, 'pub_date'=>$published);
array_push($tweet_stack, $new_tweet);
}
// sort the array in DESC order to allow the most recent tweets to be first
usort($tweet_stack, 'orderTweets');
// find the latest tweet
$latest_tweet = mysqli_real_escape_string($db, $tweet_stack[0]['content']);
// convert the date to mysql Datetime format
$latest_date = date("Y-m-d H:i:s",strtotime($tweet_stack[0]['pub_date']));
// find the total number of tweets
$num_tweets = count($tweet_stack);
// set the limit for the number of tweet records to record - change with the $tweet_record_limit var above
($num_tweets > $tweet_record_limit ? $limit = $tweet_record_limit : $limit = $num_tweets);
// search for existing records in the DB
$query = "SELECT pub_date FROM tweets";
$result = $db->query($query);
$row_count = $result->num_rows;
// if you have existing tweets
if(($row_count > 0)&&($row_count <= $limit)){
// find the most recently recorded tweet
$q4 = "SELECT MAX(pub_date) AS pub_date FROM tweets";
$r4 = $db->query($q4);
$row = $r4->fetch_assoc();
$latest_record = strtotime($row['pub_date']);
// if you have less records in the DB than you need for your application, add more records
if($row_count < $limit){
$i=$row_count; // load the iteration count with the number of existing rows
foreach($tweet_stack as $stack){
// grab the current tweet from the array
$content = mysqli_real_escape_string($db, $stack['content']);
$pub_date = date("Y-m-d H:i:s",strtotime($stack['pub_date']));
// check to see if this tweet has already been recorded
$existing = checkRecord($content, $pub_date);
if(!$existing){
insertTweet($content, $pub_date);// insert the new one
$i++; // increment the counter, ONLY if this is a new tweet
}
// stop the loop at the user defined limit
if($i==$limit){ break; }
}
updateDB($tweet_stack); // compare the DB data with the FEED to see if any Tweets have been deleted
}
// your table has reached the record limit, now see if there are any new tweets
else if(strtotime($latest_date) > $latest_record){
foreach($tweet_stack as $stack){
// grab the current tweet from the array
$content = mysqli_real_escape_string($db, $stack['content']);
$pub_date = date("Y-m-d H:i:s",strtotime($stack['pub_date']));
if(strtotime($pub_date) > $latest_record){
mail($email,'BizLab CRON Successful','The cron job has run successfully');
echo '<p>Deleting old tweets...</p>';
deleteTweet(); // this is a new tweet, now find the oldest tweet and delete it
echo '<p>Putting the new tweets...</p>';
insertTweet($content,$pub_date); // insert the new one
}
}
updateDB($tweet_stack);
}
else{
updateDB($tweet_stack);
}
}
// if you have more records in the DB than you want, remove each one over the limit
else if($row_count > $limit){
// delete the extra records beginning with the oldest
$extras = $row_count - $limit;
while($extras > 0){
// find the oldest tweet and delete it
deleteTweet();
$extras--;
}
updateDB($tweet_stack);
}
else{
// build the initial table
$i=0;
foreach($tweet_stack as $stack){
// grab the tweet data from the array and organize it for insertion into the DB
$content = mysqli_real_escape_string($db, $stack['content']);
$pub_date = date("Y-m-d H:i:s",strtotime($stack['pub_date']));
// insert the new one
insertTweet($content, $pub_date);
$i++;
// stop the loop at the user defined limit
if($i==$limit){ break; }
}
}
}
<?php
/*
// ERROR REPORTING TURNED ON
ini_set("display_startup_errors", "1");
ini_set("display_errors", "1");
error_reporting(E_ALL);
*/
require_once($_SERVER['DOCUMENT_ROOT'].'/globals/appvars.php');
function orderTweets($x, $y){
$x = strtotime($x['pub_date']);
$y = strtotime($y['pub_date']);
if($x==$y){
return 0;
}
else if($x<$y){
return 1;
}
else{
return -1;
}
}
function checkRecord($content, $pub_date){
global $db;
$query = "SELECT * FROM tweets WHERE content='$content' AND pub_date='$pub_date'";
$result = $db->query($query);
if($result->num_rows==0){
return false;
}
else{
return true;
}
}
function insertTweet($content, $pub_date){
// the current tweet is not in the DB, so insert it now
global $db;
$query = "INSERT INTO tweets (content, pub_date) VALUES('$content','$pub_date')";
$db->query($query);
}
function deleteTweet(){
global $db;
$query = "SELECT MIN(pub_date) AS pub_date FROM tweets";
$result = $db->query($query);
$row = $result->fetch_assoc();
$pub_date = $row['pub_date'];
// now delete this oldest tweet
$q2 = "DELETE FROM tweets WHERE pub_date='$pub_date' LIMIT 1";
$db->query($q2);
}
function updateDB($tweet_stack){
// get the DB records
global $db;
$query = "SELECT content, pub_date FROM tweets ORDER BY pub_date DESC";
$results = $db->query($query);
while($row = $results->fetch_assoc()){
$current_pub_date = $row['pub_date'];
if(array_search($current_pub_date, $tweet_stack)===false){
// this tweet has been deleted on Twitter, delete it in the DB
$q2 = "DELETE FROM tweets WHERE pub_date='$current_pub_date' LIMIT 1";
$db->query($q2);
$i=0; // set the iterator for the foreach loop.
foreach($tweet_stack as $stack){
if($i>0){ break; }
$content = mysqli_real_escape_string($db, $stack['content']);
$pub_date = date("Y-m-d H:i:s",strtotime($stack['pub_date']));
// check to see if this tweet has already been recorded
$existing = checkRecord($content, $pub_date);
// insert the newest tweet that isn't already in the DB
if(!$existing){
insertTweet($content, $pub_date);
$i++;
}
}
}
}
}
// set your email address here
$email = 'bill@bizlab.us';
// set this link to your personal twitter RSS feed
$url = 'http://twitter.com/statuses/user_timeline/172689882.rss';
// set your tweet record limit
// if you change this number, you will need to modify both the CSS and Javascript controlling the scroll distance
$tweet_record_limit = 5;
// initialize the array
$tweet_stack = array();
// begin your cURL request
$tw = curl_init();
curl_setopt($tw, CURLOPT_URL, $url);
curl_setopt($tw, CURLOPT_HEADER, false); // turns off HTTP headers with output
curl_setopt($tw, CURLOPT_RETURNTRANSFER, TRUE);
$data = curl_exec($tw);
curl_close($tw);
$data_string = strtolower($data); // check to be sure there were no errors
if((strpos($data_string,'over capacity')===false)
&&(strpos($data_string, 'rate limit exceeded')===false)
&&(!empty($data_string))){
// begin manipulating the ATOM FEED
$tweets = new SimpleXMLElement($data);
// loop through the array to get the tweets
foreach($tweets->channel->item as $tweet){
$published = $tweet->pubDate; // RSS DATE FORMAT: Thu, 05 Aug 2010 11:24:35 +0000
$content = trim($tweet->description);
// push data into an array
$new_tweet = array('content'=>$content, 'pub_date'=>$published);
array_push($tweet_stack, $new_tweet);
}
// sort the array in DESC order to allow the most recent tweets to be first
usort($tweet_stack, 'orderTweets');
// find the latest tweet
$latest_tweet = mysqli_real_escape_string($db, $tweet_stack[0]['content']);
// convert the date to mysql Datetime format
$latest_date = date("Y-m-d H:i:s",strtotime($tweet_stack[0]['pub_date']));
// find the total number of tweets
$num_tweets = count($tweet_stack);
// set the limit for the number of tweet records to record - change with the $tweet_record_limit var above
($num_tweets > $tweet_record_limit ? $limit = $tweet_record_limit : $limit = $num_tweets);
// search for existing records in the DB
$query = "SELECT pub_date FROM tweets";
$result = $db->query($query);
$row_count = $result->num_rows;
// if you have existing tweets
if(($row_count > 0)&&($row_count <= $limit)){
// find the most recently recorded tweet
$q4 = "SELECT MAX(pub_date) AS pub_date FROM tweets";
$r4 = $db->query($q4);
$row = $r4->fetch_assoc();
$latest_record = strtotime($row['pub_date']);
// if you have less records in the DB than you need for your application, add more records
if($row_count < $limit){
$i=$row_count; // load the iteration count with the number of existing rows
foreach($tweet_stack as $stack){
// grab the current tweet from the array
$content = mysqli_real_escape_string($db, $stack['content']);
$pub_date = date("Y-m-d H:i:s",strtotime($stack['pub_date']));
// check to see if this tweet has already been recorded
$existing = checkRecord($content, $pub_date);
if(!$existing){
insertTweet($content, $pub_date);// insert the new one
$i++; // increment the counter, ONLY if this is a new tweet
}
// stop the loop at the user defined limit
if($i==$limit){ break; }
}
updateDB($tweet_stack); // compare the DB data with the FEED to see if any Tweets have been deleted
}
// your table has reached the record limit, now see if there are any new tweets
else if(strtotime($latest_date) > $latest_record){
foreach($tweet_stack as $stack){
// grab the current tweet from the array
$content = mysqli_real_escape_string($db, $stack['content']);
$pub_date = date("Y-m-d H:i:s",strtotime($stack['pub_date']));
if(strtotime($pub_date) > $latest_record){
mail($email,'BizLab CRON Successful','The cron job has run successfully');
echo '<p>Deleting old tweets...</p>';
deleteTweet(); // this is a new tweet, now find the oldest tweet and delete it
echo '<p>Putting the new tweets...</p>';
insertTweet($content,$pub_date); // insert the new one
}
}
updateDB($tweet_stack);
}
else{
updateDB($tweet_stack);
}
}
// if you have more records in the DB than you want, remove each one over the limit
else if($row_count > $limit){
// delete the extra records beginning with the oldest
$extras = $row_count - $limit;
while($extras > 0){
// find the oldest tweet and delete it
deleteTweet();
$extras--;
}
updateDB($tweet_stack);
}
else{
// build the initial table
$i=0;
foreach($tweet_stack as $stack){
// grab the tweet data from the array and organize it for insertion into the DB
$content = mysqli_real_escape_string($db, $stack['content']);
$pub_date = date("Y-m-d H:i:s",strtotime($stack['pub_date']));
// insert the new one
insertTweet($content, $pub_date);
$i++;
// stop the loop at the user defined limit
if($i==$limit){ break; }
}
}
}
?>