diff options
Diffstat (limited to 'arch/powerpc/include/asm/book3s/64')
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/hugetlb-radix.h | 15 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/mmu-hash.h | 60 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/mmu.h | 11 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/pgtable-4k.h | 6 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/pgtable-64k.h | 6 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/pgtable.h | 99 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/tlbflush-hash.h | 5 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/tlbflush-radix.h | 20 | ||||
| -rw-r--r-- | arch/powerpc/include/asm/book3s/64/tlbflush.h | 27 |
9 files changed, 188 insertions, 61 deletions
diff --git a/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h b/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h index 60f476493..c45189aa7 100644 --- a/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h +++ b/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h @@ -11,4 +11,19 @@ extern unsigned long radix__hugetlb_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); + +static inline int hstate_get_psize(struct hstate *hstate) +{ + unsigned long shift; + + shift = huge_page_shift(hstate); + if (shift == mmu_psize_defs[MMU_PAGE_2M].shift) + return MMU_PAGE_2M; + else if (shift == mmu_psize_defs[MMU_PAGE_1G].shift) + return MMU_PAGE_1G; + else { + WARN(1, "Wrong huge page shift\n"); + return mmu_virtual_psize; + } +} #endif diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h index 74839f24f..287a656ce 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -24,6 +24,7 @@ #include <asm/book3s/64/pgtable.h> #include <asm/bug.h> #include <asm/processor.h> +#include <asm/cpu_has_feature.h> /* * SLB @@ -124,6 +125,45 @@ #ifndef __ASSEMBLY__ +struct mmu_hash_ops { + void (*hpte_invalidate)(unsigned long slot, + unsigned long vpn, + int bpsize, int apsize, + int ssize, int local); + long (*hpte_updatepp)(unsigned long slot, + unsigned long newpp, + unsigned long vpn, + int bpsize, int apsize, + int ssize, unsigned long flags); + void (*hpte_updateboltedpp)(unsigned long newpp, + unsigned long ea, + int psize, int ssize); + long (*hpte_insert)(unsigned long hpte_group, + unsigned long vpn, + unsigned long prpn, + unsigned long rflags, + unsigned long vflags, + int psize, int apsize, + int ssize); + long (*hpte_remove)(unsigned long hpte_group); + int (*hpte_removebolted)(unsigned long ea, + int psize, int ssize); + void (*flush_hash_range)(unsigned long number, int local); + void (*hugepage_invalidate)(unsigned long vsid, + unsigned long addr, + unsigned char *hpte_slot_array, + int psize, int ssize, int local); + /* + * Special for kexec. + * To be called in real mode with interrupts disabled. No locks are + * taken as such, concurrent access on pre POWER5 hardware could result + * in a deadlock. + * The linear mapping is destroyed as well. + */ + void (*hpte_clear_all)(void); +}; +extern struct mmu_hash_ops mmu_hash_ops; + struct hash_pte { __be64 v; __be64 r; @@ -151,6 +191,15 @@ static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) BUG(); } +static inline unsigned long get_sllp_encoding(int psize) +{ + unsigned long sllp; + + sllp = ((mmu_psize_defs[psize].sllp & SLB_VSID_L) >> 6) | + ((mmu_psize_defs[psize].sllp & SLB_VSID_LP) >> 4); + return sllp; +} + #endif /* __ASSEMBLY__ */ /* @@ -352,10 +401,13 @@ int htab_remove_mapping(unsigned long vstart, unsigned long vend, extern void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages); extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr); +#ifdef CONFIG_PPC_PSERIES +void hpte_init_pseries(void); +#else +static inline void hpte_init_pseries(void) { } +#endif + extern void hpte_init_native(void); -extern void hpte_init_lpar(void); -extern void hpte_init_beat(void); -extern void hpte_init_beat_v3(void); extern void slb_initialize(void); extern void slb_flush_and_rebolt(void); @@ -435,7 +487,7 @@ extern void slb_set_size(u16 size); * function. Used in slb_allocate() and do_stab_bolted. The function * computed is: (protovsid*VSID_MULTIPLIER) % VSID_MODULUS * - * rt = register continaing the proto-VSID and into which the + * rt = register containing the proto-VSID and into which the * VSID will be stored * rx = scratch register (clobbered) * diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h index 5854263d4..8afb0e00f 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu.h +++ b/arch/powerpc/include/asm/book3s/64/mmu.h @@ -23,8 +23,6 @@ struct mmu_psize_def { }; extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; -#define radix_enabled() mmu_has_feature(MMU_FTR_RADIX) - #endif /* __ASSEMBLY__ */ /* 64-bit classic hash table MMU */ @@ -102,6 +100,9 @@ extern int mmu_vmemmap_psize; extern int mmu_io_psize; /* MMU initialization */ +void mmu_early_init_devtree(void); +void hash__early_init_devtree(void); +void radix__early_init_devtree(void); extern void radix_init_native(void); extern void hash__early_init_mmu(void); extern void radix__early_init_mmu(void); @@ -127,11 +128,15 @@ extern void radix__setup_initial_memory_limit(phys_addr_t first_memblock_base, static inline void setup_initial_memory_limit(phys_addr_t first_memblock_base, phys_addr_t first_memblock_size) { - if (radix_enabled()) + if (early_radix_enabled()) return radix__setup_initial_memory_limit(first_memblock_base, first_memblock_size); return hash__setup_initial_memory_limit(first_memblock_base, first_memblock_size); } + +extern int (*register_process_table)(unsigned long base, unsigned long page_size, + unsigned long tbl_size); + #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_MMU_H_ */ diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h index 71e9abced..9db83b4e0 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h @@ -11,7 +11,7 @@ static inline int pmd_huge(pmd_t pmd) * leaf pte for huge page */ if (radix_enabled()) - return !!(pmd_val(pmd) & _PAGE_PTE); + return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); return 0; } @@ -21,7 +21,7 @@ static inline int pud_huge(pud_t pud) * leaf pte for huge page */ if (radix_enabled()) - return !!(pud_val(pud) & _PAGE_PTE); + return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); return 0; } @@ -31,7 +31,7 @@ static inline int pgd_huge(pgd_t pgd) * leaf pte for huge page */ if (radix_enabled()) - return !!(pgd_val(pgd) & _PAGE_PTE); + return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE)); return 0; } #define pgd_huge pgd_huge diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h index cb2d0a5fa..0d2845b44 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h @@ -15,7 +15,7 @@ static inline int pmd_huge(pmd_t pmd) /* * leaf pte for huge page */ - return !!(pmd_val(pmd) & _PAGE_PTE); + return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); } static inline int pud_huge(pud_t pud) @@ -23,7 +23,7 @@ static inline int pud_huge(pud_t pud) /* * leaf pte for huge page */ - return !!(pud_val(pud) & _PAGE_PTE); + return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); } static inline int pgd_huge(pgd_t pgd) @@ -31,7 +31,7 @@ static inline int pgd_huge(pgd_t pgd) /* * leaf pte for huge page */ - return !!(pgd_val(pgd) & _PAGE_PTE); + return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE)); } #define pgd_huge pgd_huge diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index ab84c89c9..263bf39ce 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -318,7 +318,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, { unsigned long old; - if ((pte_val(*ptep) & (_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) + if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) return 0; old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0); return (old & _PAGE_ACCESSED) != 0; @@ -336,8 +336,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - - if ((pte_val(*ptep) & _PAGE_WRITE) == 0) + if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0) return; pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 0); @@ -346,7 +345,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - if ((pte_val(*ptep) & _PAGE_WRITE) == 0) + if ((pte_raw(*ptep) & cpu_to_be64(_PAGE_WRITE)) == 0) return; pte_update(mm, addr, ptep, _PAGE_WRITE, 0, 1); @@ -365,17 +364,35 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, { pte_update(mm, addr, ptep, ~0UL, 0, 0); } -static inline int pte_write(pte_t pte) { return !!(pte_val(pte) & _PAGE_WRITE);} -static inline int pte_dirty(pte_t pte) { return !!(pte_val(pte) & _PAGE_DIRTY); } -static inline int pte_young(pte_t pte) { return !!(pte_val(pte) & _PAGE_ACCESSED); } -static inline int pte_special(pte_t pte) { return !!(pte_val(pte) & _PAGE_SPECIAL); } + +static inline int pte_write(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE)); +} + +static inline int pte_dirty(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_DIRTY)); +} + +static inline int pte_young(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_ACCESSED)); +} + +static inline int pte_special(pte_t pte) +{ + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SPECIAL)); +} + static inline pgprot_t pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); } #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY static inline bool pte_soft_dirty(pte_t pte) { - return !!(pte_val(pte) & _PAGE_SOFT_DIRTY); + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SOFT_DIRTY)); } + static inline pte_t pte_mksoft_dirty(pte_t pte) { return __pte(pte_val(pte) | _PAGE_SOFT_DIRTY); @@ -395,14 +412,14 @@ static inline pte_t pte_clear_soft_dirty(pte_t pte) */ static inline int pte_protnone(pte_t pte) { - return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PRIVILEGED)) == - (_PAGE_PRESENT | _PAGE_PRIVILEGED); + return (pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED)) == + cpu_to_be64(_PAGE_PRESENT | _PAGE_PRIVILEGED); } #endif /* CONFIG_NUMA_BALANCING */ static inline int pte_present(pte_t pte) { - return !!(pte_val(pte) & _PAGE_PRESENT); + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT)); } /* * Conversion functions: convert a page and protection to a page entry, @@ -474,7 +491,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) static inline bool pte_user(pte_t pte) { - return !(pte_val(pte) & _PAGE_PRIVILEGED); + return !(pte_raw(pte) & cpu_to_be64(_PAGE_PRIVILEGED)); } /* Encode and de-code a swap entry */ @@ -517,10 +534,12 @@ static inline pte_t pte_swp_mksoft_dirty(pte_t pte) { return __pte(pte_val(pte) | _PAGE_SWP_SOFT_DIRTY); } + static inline bool pte_swp_soft_dirty(pte_t pte) { - return !!(pte_val(pte) & _PAGE_SWP_SOFT_DIRTY); + return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_SOFT_DIRTY)); } + static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) { return __pte(pte_val(pte) & ~_PAGE_SWP_SOFT_DIRTY); @@ -626,8 +645,16 @@ static inline void pmd_clear(pmd_t *pmdp) *pmdp = __pmd(0); } -#define pmd_none(pmd) (!pmd_val(pmd)) -#define pmd_present(pmd) (!pmd_none(pmd)) +static inline int pmd_none(pmd_t pmd) +{ + return !pmd_raw(pmd); +} + +static inline int pmd_present(pmd_t pmd) +{ + + return !pmd_none(pmd); +} static inline int pmd_bad(pmd_t pmd) { @@ -646,19 +673,26 @@ static inline void pud_clear(pud_t *pudp) *pudp = __pud(0); } -#define pud_none(pud) (!pud_val(pud)) -#define pud_present(pud) (pud_val(pud) != 0) +static inline int pud_none(pud_t pud) +{ + return !pud_raw(pud); +} + +static inline int pud_present(pud_t pud) +{ + return !pud_none(pud); +} extern struct page *pud_page(pud_t pud); extern struct page *pmd_page(pmd_t pmd); static inline pte_t pud_pte(pud_t pud) { - return __pte(pud_val(pud)); + return __pte_raw(pud_raw(pud)); } static inline pud_t pte_pud(pte_t pte) { - return __pud(pte_val(pte)); + return __pud_raw(pte_raw(pte)); } #define pud_write(pud) pte_write(pud_pte(pud)) @@ -681,17 +715,24 @@ static inline void pgd_clear(pgd_t *pgdp) *pgdp = __pgd(0); } -#define pgd_none(pgd) (!pgd_val(pgd)) -#define pgd_present(pgd) (!pgd_none(pgd)) +static inline int pgd_none(pgd_t pgd) +{ + return !pgd_raw(pgd); +} + +static inline int pgd_present(pgd_t pgd) +{ + return !pgd_none(pgd); +} static inline pte_t pgd_pte(pgd_t pgd) { - return __pte(pgd_val(pgd)); + return __pte_raw(pgd_raw(pgd)); } static inline pgd_t pte_pgd(pte_t pte) { - return __pgd(pte_val(pte)); + return __pgd_raw(pte_raw(pte)); } static inline int pgd_bad(pgd_t pgd) @@ -783,12 +824,12 @@ struct page *realmode_pfn_to_page(unsigned long pfn); static inline pte_t pmd_pte(pmd_t pmd) { - return __pte(pmd_val(pmd)); + return __pte_raw(pmd_raw(pmd)); } static inline pmd_t pte_pmd(pte_t pte) { - return __pmd(pte_val(pte)); + return __pmd_raw(pte_raw(pte)); } static inline pte_t *pmdp_ptep(pmd_t *pmd) @@ -849,7 +890,7 @@ pmd_hugepage_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, static inline int pmd_large(pmd_t pmd) { - return !!(pmd_val(pmd) & _PAGE_PTE); + return !!(pmd_raw(pmd) & cpu_to_be64(_PAGE_PTE)); } static inline pmd_t pmd_mknotpresent(pmd_t pmd) @@ -865,7 +906,7 @@ static inline int __pmdp_test_and_clear_young(struct mm_struct *mm, { unsigned long old; - if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) + if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_ACCESSED | H_PAGE_HASHPTE)) == 0) return 0; old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0); return ((old & _PAGE_ACCESSED) != 0); @@ -876,7 +917,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { - if ((pmd_val(*pmdp) & _PAGE_WRITE) == 0) + if ((pmd_raw(*pmdp) & cpu_to_be64(_PAGE_WRITE)) == 0) return; pmd_hugepage_update(mm, addr, pmdp, _PAGE_WRITE, 0); diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h index f12ddf5e8..2f6373144 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h @@ -75,11 +75,6 @@ static inline void hash__flush_tlb_page(struct vm_area_struct *vma, { } -static inline void hash__flush_tlb_page_nohash(struct vm_area_struct *vma, - unsigned long vmaddr) -{ -} - static inline void hash__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index 3fa94fcac..65037762b 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -10,27 +10,35 @@ static inline int mmu_get_ap(int psize) return mmu_psize_defs[psize].ap; } +extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end); +extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, + unsigned long end, int psize); +extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end); extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end); extern void radix__local_flush_tlb_mm(struct mm_struct *mm); extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); -extern void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, - unsigned long ap, int nid); extern void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); +extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, + int psize); extern void radix__tlb_flush(struct mmu_gather *tlb); #ifdef CONFIG_SMP extern void radix__flush_tlb_mm(struct mm_struct *mm); extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); -extern void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, - unsigned long ap, int nid); extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); +extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, + int psize); #else #define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm) #define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr) -#define radix___flush_tlb_page(mm,addr,p,i) radix___local_flush_tlb_page(mm,addr,p,i) +#define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p) #define radix__flush_tlb_pwc(tlb, addr) radix__local_flush_tlb_pwc(tlb, addr) #endif - +extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa, + unsigned long page_size); +extern void radix__flush_tlb_lpid(unsigned long lpid); #endif diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index 96e5769b1..72b925f97 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -7,6 +7,25 @@ #include <asm/book3s/64/tlbflush-hash.h> #include <asm/book3s/64/tlbflush-radix.h> +#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE +static inline void flush_pmd_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + if (radix_enabled()) + return radix__flush_pmd_tlb_range(vma, start, end); + return hash__flush_tlb_range(vma, start, end); +} + +#define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE +static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + if (radix_enabled()) + return radix__flush_hugetlb_tlb_range(vma, start, end); + return hash__flush_tlb_range(vma, start, end); +} + static inline void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { @@ -38,14 +57,6 @@ static inline void local_flush_tlb_page(struct vm_area_struct *vma, return hash__local_flush_tlb_page(vma, vmaddr); } -static inline void flush_tlb_page_nohash(struct vm_area_struct *vma, - unsigned long vmaddr) -{ - if (radix_enabled()) - return radix__flush_tlb_page(vma, vmaddr); - return hash__flush_tlb_page_nohash(vma, vmaddr); -} - static inline void tlb_flush(struct mmu_gather *tlb) { if (radix_enabled()) |