2017-11-24 15:00:36 +01:00
// SPDX-License-Identifier: GPL-2.0
2013-04-16 14:14:44 +02:00
/*
* s390 specific pci instructions
*
* Copyright IBM Corp . 2013
*/
# include <linux/export.h>
# include <linux/errno.h>
# include <linux/delay.h>
2019-04-14 15:38:01 +02:00
# include <linux/jump_label.h>
2022-02-28 11:22:12 +01:00
# include <asm/asm-extable.h>
2017-10-30 14:38:58 +01:00
# include <asm/facility.h>
2013-04-16 14:14:44 +02:00
# include <asm/pci_insn.h>
2015-08-18 19:39:27 +02:00
# include <asm/pci_debug.h>
2019-04-14 16:25:54 +02:00
# include <asm/pci_io.h>
2013-04-16 14:16:14 +02:00
# include <asm/processor.h>
2013-04-16 14:14:44 +02:00
# define ZPCI_INSN_BUSY_DELAY 1 /* 1 microsecond */
2022-02-24 15:45:33 +01:00
struct zpci_err_insn_data {
u8 insn ;
u8 cc ;
u8 status ;
union {
struct {
u64 req ;
u64 offset ;
} ;
struct {
u64 addr ;
u64 len ;
} ;
} ;
} __packed ;
2022-02-25 09:45:24 +01:00
static inline void zpci_err_insn_req ( int lvl , u8 insn , u8 cc , u8 status ,
2022-02-24 15:45:33 +01:00
u64 req , u64 offset )
{
struct zpci_err_insn_data data = {
. insn = insn , . cc = cc , . status = status ,
. req = req , . offset = offset } ;
2022-02-25 09:45:24 +01:00
zpci_err_hex_level ( lvl , & data , sizeof ( data ) ) ;
2022-02-24 15:45:33 +01:00
}
2022-02-25 09:45:24 +01:00
static inline void zpci_err_insn_addr ( int lvl , u8 insn , u8 cc , u8 status ,
2022-02-24 15:45:33 +01:00
u64 addr , u64 len )
2015-08-18 19:39:27 +02:00
{
2022-02-24 15:45:33 +01:00
struct zpci_err_insn_data data = {
. insn = insn , . cc = cc , . status = status ,
. addr = addr , . len = len } ;
2015-08-18 19:39:27 +02:00
2022-02-25 09:45:24 +01:00
zpci_err_hex_level ( lvl , & data , sizeof ( data ) ) ;
2015-08-18 19:39:27 +02:00
}
2013-04-16 14:14:44 +02:00
/* Modify PCI Function Controls */
static inline u8 __mpcifc ( u64 req , struct zpci_fib * fib , u8 * status )
{
u8 cc ;
asm volatile (
" .insn rxy,0xe300000000d0,%[req],%[fib] \n "
" ipm %[cc] \n "
" srl %[cc],28 \n "
: [ cc ] " =d " ( cc ) , [ req ] " +d " ( req ) , [ fib ] " +Q " ( * fib )
: : " cc " ) ;
* status = req > > 24 & 0xff ;
return cc ;
}
2017-06-10 13:54:44 +02:00
u8 zpci_mod_fc ( u64 req , struct zpci_fib * fib , u8 * status )
2013-04-16 14:14:44 +02:00
{
2022-02-25 09:45:24 +01:00
bool retried = false ;
2017-06-10 13:54:44 +02:00
u8 cc ;
2013-04-16 14:14:44 +02:00
do {
2017-06-10 13:54:44 +02:00
cc = __mpcifc ( req , fib , status ) ;
2022-02-25 09:45:24 +01:00
if ( cc = = 2 ) {
2013-04-16 14:14:44 +02:00
msleep ( ZPCI_INSN_BUSY_DELAY ) ;
2022-02-25 09:45:24 +01:00
if ( ! retried ) {
zpci_err_insn_req ( 1 , ' M ' , cc , * status , req , 0 ) ;
retried = true ;
}
}
2013-04-16 14:14:44 +02:00
} while ( cc = = 2 ) ;
if ( cc )
2022-02-25 09:45:24 +01:00
zpci_err_insn_req ( 0 , ' M ' , cc , * status , req , 0 ) ;
else if ( retried )
zpci_err_insn_req ( 1 , ' M ' , cc , * status , req , 0 ) ;
2015-08-18 19:39:27 +02:00
2017-06-10 13:54:44 +02:00
return cc ;
2013-04-16 14:14:44 +02:00
}
2022-06-06 16:33:19 -04:00
EXPORT_SYMBOL_GPL ( zpci_mod_fc ) ;
2013-04-16 14:14:44 +02:00
/* Refresh PCI Translations */
static inline u8 __rpcit ( u64 fn , u64 addr , u64 range , u8 * status )
{
2021-06-15 17:08:23 +02:00
union register_pair addr_range = { . even = addr , . odd = range } ;
2013-04-16 14:14:44 +02:00
u8 cc ;
asm volatile (
2021-06-15 17:08:23 +02:00
" .insn rre,0xb9d30000,%[fn],%[addr_range] \n "
2013-04-16 14:14:44 +02:00
" ipm %[cc] \n "
" srl %[cc],28 \n "
: [ cc ] " =d " ( cc ) , [ fn ] " +d " ( fn )
2021-06-15 17:08:23 +02:00
: [ addr_range ] " d " ( addr_range . pair )
2013-04-16 14:14:44 +02:00
: " cc " ) ;
* status = fn > > 24 & 0xff ;
return cc ;
}
2013-06-25 14:52:23 +02:00
int zpci_refresh_trans ( u64 fn , u64 addr , u64 range )
2013-04-16 14:14:44 +02:00
{
2022-02-25 09:45:24 +01:00
bool retried = false ;
2013-04-16 14:14:44 +02:00
u8 cc , status ;
do {
cc = __rpcit ( fn , addr , range , & status ) ;
2022-02-25 09:45:24 +01:00
if ( cc = = 2 ) {
2013-04-16 14:14:44 +02:00
udelay ( ZPCI_INSN_BUSY_DELAY ) ;
2022-02-25 09:45:24 +01:00
if ( ! retried ) {
zpci_err_insn_addr ( 1 , ' R ' , cc , status , addr , range ) ;
retried = true ;
}
}
2013-04-16 14:14:44 +02:00
} while ( cc = = 2 ) ;
if ( cc )
2022-02-25 09:45:24 +01:00
zpci_err_insn_addr ( 0 , ' R ' , cc , status , addr , range ) ;
else if ( retried )
zpci_err_insn_addr ( 1 , ' R ' , cc , status , addr , range ) ;
2015-08-18 19:39:27 +02:00
2017-12-01 18:47:32 +01:00
if ( cc = = 1 & & ( status = = 4 | | status = = 16 ) )
return - ENOMEM ;
2013-04-16 14:14:44 +02:00
return ( cc ) ? - EIO : 0 ;
}
/* Set Interruption Controls */
2022-06-06 16:33:11 -04:00
int zpci_set_irq_ctrl ( u16 ctl , u8 isc , union zpci_sic_iib * iib )
2013-04-16 14:14:44 +02:00
{
2017-10-30 14:38:58 +01:00
if ( ! test_facility ( 72 ) )
return - EIO ;
2018-09-27 13:57:12 +02:00
asm volatile (
" .insn rsy,0xeb00000000d1,%[ctl],%[isc],%[iib] \n "
: : [ ctl ] " d " ( ctl ) , [ isc ] " d " ( isc < < 27 ) , [ iib ] " Q " ( * iib ) ) ;
2017-10-30 14:38:58 +01:00
return 0 ;
2013-04-16 14:14:44 +02:00
}
2022-06-06 16:33:11 -04:00
EXPORT_SYMBOL_GPL ( zpci_set_irq_ctrl ) ;
2013-04-16 14:14:44 +02:00
/* PCI Load */
2016-06-20 14:03:25 +02:00
static inline int ____pcilg ( u64 * data , u64 req , u64 offset , u8 * status )
2013-04-16 14:14:44 +02:00
{
2021-06-15 17:08:23 +02:00
union register_pair req_off = { . even = req , . odd = offset } ;
2013-04-16 14:16:14 +02:00
int cc = - ENXIO ;
2013-04-16 14:14:44 +02:00
u64 __data ;
asm volatile (
2021-06-15 17:08:23 +02:00
" .insn rre,0xb9d20000,%[data],%[req_off] \n "
2013-04-16 14:16:14 +02:00
" 0: ipm %[cc] \n "
2013-04-16 14:14:44 +02:00
" srl %[cc],28 \n "
2013-04-16 14:16:14 +02:00
" 1: \n "
EX_TABLE ( 0 b , 1 b )
2021-06-15 17:08:23 +02:00
: [ cc ] " +d " ( cc ) , [ data ] " =d " ( __data ) ,
[ req_off ] " +&d " ( req_off . pair ) : : " cc " ) ;
* status = req_off . even > > 24 & 0xff ;
2016-06-20 14:03:25 +02:00
* data = __data ;
return cc ;
}
static inline int __pcilg ( u64 * data , u64 req , u64 offset , u8 * status )
{
u64 __data ;
int cc ;
cc = ____pcilg ( & __data , req , offset , status ) ;
2013-04-16 14:17:15 +02:00
if ( ! cc )
* data = __data ;
2013-04-16 14:14:44 +02:00
return cc ;
}
2019-04-14 16:25:54 +02:00
int __zpci_load ( u64 * data , u64 req , u64 offset )
2013-04-16 14:14:44 +02:00
{
2022-02-25 09:45:24 +01:00
bool retried = false ;
2013-04-16 14:16:14 +02:00
u8 status ;
int cc ;
2013-04-16 14:14:44 +02:00
do {
cc = __pcilg ( data , req , offset , & status ) ;
2022-02-25 09:45:24 +01:00
if ( cc = = 2 ) {
2013-04-16 14:14:44 +02:00
udelay ( ZPCI_INSN_BUSY_DELAY ) ;
2022-02-25 09:45:24 +01:00
if ( ! retried ) {
zpci_err_insn_req ( 1 , ' l ' , cc , status , req , offset ) ;
retried = true ;
}
}
2013-04-16 14:14:44 +02:00
} while ( cc = = 2 ) ;
2013-04-16 14:16:14 +02:00
if ( cc )
2022-02-25 09:45:24 +01:00
zpci_err_insn_req ( 0 , ' l ' , cc , status , req , offset ) ;
else if ( retried )
zpci_err_insn_req ( 1 , ' l ' , cc , status , req , offset ) ;
2015-08-18 19:39:27 +02:00
2013-04-16 14:16:14 +02:00
return ( cc > 0 ) ? - EIO : cc ;
2013-04-16 14:14:44 +02:00
}
2019-04-14 16:25:54 +02:00
EXPORT_SYMBOL_GPL ( __zpci_load ) ;
2019-04-14 15:38:01 +02:00
static inline int zpci_load_fh ( u64 * data , const volatile void __iomem * addr ,
unsigned long len )
2019-04-14 16:25:54 +02:00
{
struct zpci_iomap_entry * entry = & zpci_iomap_start [ ZPCI_IDX ( addr ) ] ;
2021-07-07 10:42:43 +02:00
u64 req = ZPCI_CREATE_REQ ( READ_ONCE ( entry - > fh ) , entry - > bar , len ) ;
2019-04-14 16:25:54 +02:00
return __zpci_load ( data , req , ZPCI_OFFSET ( addr ) ) ;
}
2019-04-14 15:38:01 +02:00
static inline int __pcilg_mio ( u64 * data , u64 ioaddr , u64 len , u8 * status )
{
2021-06-15 17:08:23 +02:00
union register_pair ioaddr_len = { . even = ioaddr , . odd = len } ;
2019-04-14 15:38:01 +02:00
int cc = - ENXIO ;
u64 __data ;
asm volatile (
2021-06-15 17:08:23 +02:00
" .insn rre,0xb9d60000,%[data],%[ioaddr_len] \n "
2019-04-14 15:38:01 +02:00
" 0: ipm %[cc] \n "
" srl %[cc],28 \n "
" 1: \n "
EX_TABLE ( 0 b , 1 b )
2021-06-15 17:08:23 +02:00
: [ cc ] " +d " ( cc ) , [ data ] " =d " ( __data ) ,
[ ioaddr_len ] " +&d " ( ioaddr_len . pair ) : : " cc " ) ;
* status = ioaddr_len . odd > > 24 & 0xff ;
2019-04-14 15:38:01 +02:00
* data = __data ;
return cc ;
}
int zpci_load ( u64 * data , const volatile void __iomem * addr , unsigned long len )
{
u8 status ;
int cc ;
if ( ! static_branch_unlikely ( & have_mio ) )
return zpci_load_fh ( data , addr , len ) ;
cc = __pcilg_mio ( data , ( __force u64 ) addr , len , & status ) ;
if ( cc )
2022-02-25 09:45:24 +01:00
zpci_err_insn_addr ( 0 , ' L ' , cc , status , ( __force u64 ) addr , len ) ;
2019-04-14 15:38:01 +02:00
return ( cc > 0 ) ? - EIO : cc ;
}
2013-06-25 14:52:23 +02:00
EXPORT_SYMBOL_GPL ( zpci_load ) ;
2013-04-16 14:14:44 +02:00
/* PCI Store */
2013-04-16 14:16:14 +02:00
static inline int __pcistg ( u64 data , u64 req , u64 offset , u8 * status )
2013-04-16 14:14:44 +02:00
{
2021-06-15 17:08:23 +02:00
union register_pair req_off = { . even = req , . odd = offset } ;
2013-04-16 14:16:14 +02:00
int cc = - ENXIO ;
2013-04-16 14:14:44 +02:00
asm volatile (
2021-06-15 17:08:23 +02:00
" .insn rre,0xb9d00000,%[data],%[req_off] \n "
2013-04-16 14:16:14 +02:00
" 0: ipm %[cc] \n "
2013-04-16 14:14:44 +02:00
" srl %[cc],28 \n "
2013-04-16 14:16:14 +02:00
" 1: \n "
EX_TABLE ( 0 b , 1 b )
2021-06-15 17:08:23 +02:00
: [ cc ] " +d " ( cc ) , [ req_off ] " +&d " ( req_off . pair )
: [ data ] " d " ( data )
2013-04-16 14:14:44 +02:00
: " cc " ) ;
2021-06-15 17:08:23 +02:00
* status = req_off . even > > 24 & 0xff ;
2013-04-16 14:14:44 +02:00
return cc ;
}
2019-04-14 16:25:54 +02:00
int __zpci_store ( u64 data , u64 req , u64 offset )
2013-04-16 14:14:44 +02:00
{
2022-02-25 09:45:24 +01:00
bool retried = false ;
2013-04-16 14:16:14 +02:00
u8 status ;
int cc ;
2013-04-16 14:14:44 +02:00
do {
cc = __pcistg ( data , req , offset , & status ) ;
2022-02-25 09:45:24 +01:00
if ( cc = = 2 ) {
2013-04-16 14:14:44 +02:00
udelay ( ZPCI_INSN_BUSY_DELAY ) ;
2022-02-25 09:45:24 +01:00
if ( ! retried ) {
zpci_err_insn_req ( 1 , ' s ' , cc , status , req , offset ) ;
retried = true ;
}
}
2013-04-16 14:14:44 +02:00
} while ( cc = = 2 ) ;
if ( cc )
2022-02-25 09:45:24 +01:00
zpci_err_insn_req ( 0 , ' s ' , cc , status , req , offset ) ;
else if ( retried )
zpci_err_insn_req ( 1 , ' s ' , cc , status , req , offset ) ;
2015-08-18 19:39:27 +02:00
2013-04-16 14:16:14 +02:00
return ( cc > 0 ) ? - EIO : cc ;
2013-04-16 14:14:44 +02:00
}
2019-04-14 16:25:54 +02:00
EXPORT_SYMBOL_GPL ( __zpci_store ) ;
2019-04-14 15:38:01 +02:00
static inline int zpci_store_fh ( const volatile void __iomem * addr , u64 data ,
unsigned long len )
2019-04-14 16:25:54 +02:00
{
struct zpci_iomap_entry * entry = & zpci_iomap_start [ ZPCI_IDX ( addr ) ] ;
2021-07-07 10:42:43 +02:00
u64 req = ZPCI_CREATE_REQ ( READ_ONCE ( entry - > fh ) , entry - > bar , len ) ;
2019-04-14 16:25:54 +02:00
return __zpci_store ( data , req , ZPCI_OFFSET ( addr ) ) ;
}
2019-04-14 15:38:01 +02:00
static inline int __pcistg_mio ( u64 data , u64 ioaddr , u64 len , u8 * status )
{
2021-06-15 17:08:23 +02:00
union register_pair ioaddr_len = { . even = ioaddr , . odd = len } ;
2019-04-14 15:38:01 +02:00
int cc = - ENXIO ;
asm volatile (
2021-06-15 17:08:23 +02:00
" .insn rre,0xb9d40000,%[data],%[ioaddr_len] \n "
2019-04-14 15:38:01 +02:00
" 0: ipm %[cc] \n "
" srl %[cc],28 \n "
" 1: \n "
EX_TABLE ( 0 b , 1 b )
2021-06-15 17:08:23 +02:00
: [ cc ] " +d " ( cc ) , [ ioaddr_len ] " +&d " ( ioaddr_len . pair )
: [ data ] " d " ( data )
: " cc " , " memory " ) ;
* status = ioaddr_len . odd > > 24 & 0xff ;
2019-04-14 15:38:01 +02:00
return cc ;
}
int zpci_store ( const volatile void __iomem * addr , u64 data , unsigned long len )
{
u8 status ;
int cc ;
if ( ! static_branch_unlikely ( & have_mio ) )
return zpci_store_fh ( addr , data , len ) ;
cc = __pcistg_mio ( data , ( __force u64 ) addr , len , & status ) ;
if ( cc )
2022-02-25 09:45:24 +01:00
zpci_err_insn_addr ( 0 , ' S ' , cc , status , ( __force u64 ) addr , len ) ;
2019-04-14 15:38:01 +02:00
return ( cc > 0 ) ? - EIO : cc ;
}
2013-06-25 14:52:23 +02:00
EXPORT_SYMBOL_GPL ( zpci_store ) ;
2013-04-16 14:14:44 +02:00
/* PCI Store Block */
2013-04-16 14:16:14 +02:00
static inline int __pcistb ( const u64 * data , u64 req , u64 offset , u8 * status )
2013-04-16 14:14:44 +02:00
{
2013-04-16 14:16:14 +02:00
int cc = - ENXIO ;
2013-04-16 14:14:44 +02:00
asm volatile (
" .insn rsy,0xeb00000000d0,%[req],%[offset],%[data] \n "
2013-04-16 14:16:14 +02:00
" 0: ipm %[cc] \n "
2013-04-16 14:14:44 +02:00
" srl %[cc],28 \n "
2013-04-16 14:16:14 +02:00
" 1: \n "
EX_TABLE ( 0 b , 1 b )
: [ cc ] " +d " ( cc ) , [ req ] " +d " ( req )
2013-04-16 14:14:44 +02:00
: [ offset ] " d " ( offset ) , [ data ] " Q " ( * data )
: " cc " ) ;
* status = req > > 24 & 0xff ;
return cc ;
}
2019-04-14 16:25:54 +02:00
int __zpci_store_block ( const u64 * data , u64 req , u64 offset )
2013-04-16 14:14:44 +02:00
{
2022-02-25 09:45:24 +01:00
bool retried = false ;
2013-04-16 14:16:14 +02:00
u8 status ;
int cc ;
2013-04-16 14:14:44 +02:00
do {
cc = __pcistb ( data , req , offset , & status ) ;
2022-02-25 09:45:24 +01:00
if ( cc = = 2 ) {
2013-04-16 14:14:44 +02:00
udelay ( ZPCI_INSN_BUSY_DELAY ) ;
2022-02-25 09:45:24 +01:00
if ( ! retried ) {
zpci_err_insn_req ( 0 , ' b ' , cc , status , req , offset ) ;
retried = true ;
}
}
2013-04-16 14:14:44 +02:00
} while ( cc = = 2 ) ;
if ( cc )
2022-02-25 09:45:24 +01:00
zpci_err_insn_req ( 0 , ' b ' , cc , status , req , offset ) ;
else if ( retried )
zpci_err_insn_req ( 1 , ' b ' , cc , status , req , offset ) ;
2015-08-18 19:39:27 +02:00
2013-04-16 14:16:14 +02:00
return ( cc > 0 ) ? - EIO : cc ;
2013-04-16 14:14:44 +02:00
}
2019-04-14 16:25:54 +02:00
EXPORT_SYMBOL_GPL ( __zpci_store_block ) ;
2019-04-14 15:38:01 +02:00
static inline int zpci_write_block_fh ( volatile void __iomem * dst ,
const void * src , unsigned long len )
2019-04-14 16:25:54 +02:00
{
struct zpci_iomap_entry * entry = & zpci_iomap_start [ ZPCI_IDX ( dst ) ] ;
u64 req = ZPCI_CREATE_REQ ( entry - > fh , entry - > bar , len ) ;
u64 offset = ZPCI_OFFSET ( dst ) ;
return __zpci_store_block ( src , req , offset ) ;
}
2019-04-14 15:38:01 +02:00
static inline int __pcistb_mio ( const u64 * data , u64 ioaddr , u64 len , u8 * status )
{
int cc = - ENXIO ;
asm volatile (
" .insn rsy,0xeb00000000d4,%[len],%[ioaddr],%[data] \n "
" 0: ipm %[cc] \n "
" srl %[cc],28 \n "
" 1: \n "
EX_TABLE ( 0 b , 1 b )
: [ cc ] " +d " ( cc ) , [ len ] " +d " ( len )
: [ ioaddr ] " d " ( ioaddr ) , [ data ] " Q " ( * data )
: " cc " ) ;
* status = len > > 24 & 0xff ;
return cc ;
}
int zpci_write_block ( volatile void __iomem * dst ,
const void * src , unsigned long len )
{
u8 status ;
int cc ;
if ( ! static_branch_unlikely ( & have_mio ) )
return zpci_write_block_fh ( dst , src , len ) ;
cc = __pcistb_mio ( src , ( __force u64 ) dst , len , & status ) ;
if ( cc )
2022-02-25 09:45:24 +01:00
zpci_err_insn_addr ( 0 , ' B ' , cc , status , ( __force u64 ) dst , len ) ;
2019-04-14 15:38:01 +02:00
return ( cc > 0 ) ? - EIO : cc ;
}
2019-04-14 16:25:54 +02:00
EXPORT_SYMBOL_GPL ( zpci_write_block ) ;
2019-04-14 15:38:01 +02:00
static inline void __pciwb_mio ( void )
{
2022-01-05 16:10:54 +01:00
asm volatile ( " .insn rre,0xb9d50000,0,0 \n " ) ;
2019-04-14 15:38:01 +02:00
}
void zpci_barrier ( void )
{
if ( static_branch_likely ( & have_mio ) )
__pciwb_mio ( ) ;
}
EXPORT_SYMBOL_GPL ( zpci_barrier ) ;