I guess you are using web3.js to send transaction calling your function.
This does not support return values.
You can get return value only when calling this function from another contract or calling it as ctr.payout.call() (note that in the latter case, the function can't change contract's state).
Please see these questions for more information:
It is possible to use events to track return values.
Here is test contract and example geth session:
x.sol
pragma solidity ^0.4.0;
contract Test {
bool public projectPaid = false;
uint public amountRaised = 0;
function payout() returns (uint) {
if (projectPaid) return 3;
projectPaid = true;
if (!msg.sender.send(amountRaised)) {
projectPaid = false;
return 4;
}
return 5;
}
}
geth console
var x_sol_testContract = web3.eth.contract([{"constant":false,"inputs":[],"name":"payout","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"amountRaised","outputs":[{"name":"","type":"uint256"}],"payable":false,"type":"function"},{"constant":true,"inputs":[],"name":"projectPaid","outputs":[{"name":"","type":"bool"}],"payable":false,"type":"function"}]);
var x_sol_test = x_sol_testContract.new(
{
from: web3.eth.accounts[0],
data: '0x60606040526000805460ff19168155600155341561001957fe5b5b61014d806100296000396000f300606060405263ffffffff60e060020a60003504166363bd1d4a81146100375780637b3e5e7b14610059578063c2e39a731461007b575bfe5b341561003f57fe5b61004761009f565b60408051918252519081900360200190f35b341561006157fe5b610047610112565b60408051918252519081900360200190f35b341561008357fe5b61008b610118565b604080519115158252519081900360200190f35b6000805460ff16156100b35750600361010f565b6000805460ff1916600190811782555460405173ffffffffffffffffffffffffffffffffffffffff33169282156108fc02929190818181858888f19350505050151561010b57506000805460ff19169055600461010f565b5060055b90565b60015481565b60005460ff16815600a165627a7a723058205acabc04fdd5e478a5e40215426abb624ec48fea0ef750e4e6a0f14cb328dbe90029',
gas: '4700000'
})
x_sol_test.payout.call()
Try to run this and see if it returns 5. I my case it does.